diff options
Diffstat (limited to 'controller-server/src/main/java')
65 files changed, 1203 insertions, 1329 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index e3400d76bce..ae2de96f511 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -7,14 +7,16 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Change.VersionChange; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.Collections; @@ -38,38 +40,44 @@ public class Application { private final ApplicationId id; private final DeploymentSpec deploymentSpec; private final ValidationOverrides validationOverrides; - private final Map<Zone, Deployment> deployments; + private final Map<ZoneId, Deployment> deployments; private final DeploymentJobs deploymentJobs; private final Optional<Change> deploying; private final boolean outstandingChange; private final Optional<IssueId> ownershipIssueId; private final ApplicationMetrics metrics; + private final Optional<RotationId> rotation; /** Creates an empty application */ public Application(ApplicationId id) { - this(id, DeploymentSpec.empty, ValidationOverrides.empty, ImmutableMap.of(), + this(id, DeploymentSpec.empty, ValidationOverrides.empty, Collections.emptyMap(), new DeploymentJobs(Optional.empty(), Collections.emptyList(), Optional.empty()), - Optional.empty(), false, Optional.empty(), new ApplicationMetrics(0, 0)); + Optional.empty(), false, Optional.empty(), new ApplicationMetrics(0, 0), + Optional.empty()); } /** Used from persistence layer: Do not use */ public Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, List<Deployment> deployments, DeploymentJobs deploymentJobs, Optional<Change> deploying, - boolean outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics) { + boolean outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics, + Optional<RotationId> rotation) { this(id, deploymentSpec, validationOverrides, deployments.stream().collect(Collectors.toMap(Deployment::zone, d -> d)), - deploymentJobs, deploying, outstandingChange, ownershipIssueId, metrics); + deploymentJobs, deploying, outstandingChange, ownershipIssueId, metrics, rotation); } Application(ApplicationId id, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, - Map<Zone, Deployment> deployments, DeploymentJobs deploymentJobs, Optional<Change> deploying, - boolean outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics) { + Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Optional<Change> deploying, + boolean outstandingChange, Optional<IssueId> ownershipIssueId, ApplicationMetrics metrics, + Optional<RotationId> rotation) { Objects.requireNonNull(id, "id cannot be null"); Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); Objects.requireNonNull(validationOverrides, "validationOverrides cannot be null"); Objects.requireNonNull(deployments, "deployments cannot be null"); Objects.requireNonNull(deploymentJobs, "deploymentJobs cannot be null"); Objects.requireNonNull(deploying, "deploying cannot be null"); + Objects.requireNonNull(metrics, "metrics cannot be null"); + Objects.requireNonNull(rotation, "rotation cannot be null"); this.id = id; this.deploymentSpec = deploymentSpec; this.validationOverrides = validationOverrides; @@ -79,6 +87,7 @@ public class Application { this.outstandingChange = outstandingChange; this.ownershipIssueId = ownershipIssueId; this.metrics = metrics; + this.rotation = rotation; } public ApplicationId id() { return id; } @@ -97,13 +106,13 @@ public class Application { public ValidationOverrides validationOverrides() { return validationOverrides; } /** Returns an immutable map of the current deployments of this */ - public Map<Zone, Deployment> deployments() { return deployments; } + public Map<ZoneId, Deployment> deployments() { return deployments; } /** * Returns an immutable map of the current *production* deployments of this * (deployments also includes manually deployed environments) */ - public Map<Zone, Deployment> productionDeployments() { + public Map<ZoneId, Deployment> productionDeployments() { return ImmutableMap.copyOf(deployments.values().stream() .filter(deployment -> deployment.zone().environment() == Environment.prod) .collect(Collectors.toMap(Deployment::zone, Function.identity()))); @@ -142,7 +151,7 @@ public class Application { } /** Returns the version a new deployment to this zone should use for this application */ - public Version deployVersionIn(Zone zone, Controller controller) { + public Version deployVersionIn(ZoneId zone, Controller controller) { if (deploying().isPresent() && deploying().get() instanceof VersionChange) return ((Change.VersionChange) deploying().get()).version(); @@ -150,13 +159,13 @@ public class Application { } /** Returns the current version this application has, or if none; should use, in the given zone */ - public Version versionIn(Zone zone, Controller controller) { + public Version versionIn(ZoneId zone, Controller controller) { return Optional.ofNullable(deployments().get(zone)).map(Deployment::version) // Already deployed in this zone: Use that version .orElse(oldestDeployedVersion().orElse(controller.systemVersion())); } /** Returns the revision a new deployment to this zone should use for this application, or empty if we don't know */ - public Optional<ApplicationRevision> deployRevisionIn(Zone zone) { + public Optional<ApplicationRevision> deployRevisionIn(ZoneId zone) { if (deploying().isPresent() && deploying().get() instanceof Change.ApplicationChange) return ((Change.ApplicationChange) deploying().get()).revision(); @@ -164,10 +173,15 @@ public class Application { } /** Returns the revision this application is or should be deployed with in the given zone, or empty if unknown. */ - public Optional<ApplicationRevision> revisionIn(Zone zone) { + public Optional<ApplicationRevision> revisionIn(ZoneId zone) { return Optional.ofNullable(deployments().get(zone)).map(Deployment::revision); } + /** Returns the global rotation of this, if present */ + public Optional<ApplicationRotation> rotation() { + return rotation.map(rotation -> new ApplicationRotation(id, rotation)); + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 841b9b4dd9f..fb92109c5df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -9,9 +8,9 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.ApplicationAlias; import com.yahoo.vespa.hosted.controller.api.InstanceEndpoints; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; @@ -29,27 +28,33 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstance import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; -import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; +import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.maintenance.DeploymentExpirer; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.rotation.RotationRepository; +import com.yahoo.vespa.hosted.controller.rotation.Rotation; +import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import com.yahoo.yolean.Exceptions; import java.io.IOException; import java.net.URI; @@ -82,6 +87,7 @@ public class ApplicationController { /** For permanent storage */ private final ControllerDb db; + /** For working memory storage and sharing between controllers */ private final CuratorDb curator; @@ -93,26 +99,26 @@ public class ApplicationController { private final Clock clock; private final DeploymentTrigger deploymentTrigger; - + ApplicationController(Controller controller, ControllerDb db, CuratorDb curator, - RotationRepository rotationRepository, - AthenzClientFactory zmsClientFactory, + AthenzClientFactory zmsClientFactory, RotationsConfig rotationsConfig, NameService nameService, ConfigServerClient configserverClient, RoutingGenerator routingGenerator, Clock clock) { this.controller = controller; this.db = db; this.curator = curator; - this.rotationRepository = rotationRepository; this.zmsClientFactory = zmsClientFactory; this.nameService = nameService; this.configserverClient = configserverClient; this.routingGenerator = routingGenerator; this.clock = clock; + this.rotationRepository = new RotationRepository(rotationsConfig, this, curator); this.deploymentTrigger = new DeploymentTrigger(controller, curator, clock); - for (Application application : db.listApplications()) - lockedIfPresent(application.id(), this::store); + for (Application application : db.listApplications()) { + lockIfPresent(application.id(), this::store); + } } /** Returns the application with the given id, or null if it is not present */ @@ -231,10 +237,6 @@ public class ApplicationController { if ( ! (id.instance().value().equals("default") || id.instance().value().startsWith("default-pr"))) // TODO: Support instances properly throw new UnsupportedOperationException("Only the instance names 'default' and names starting with 'default-pr' are supported at the moment"); try (Lock lock = lock(id)) { - // TODO: Throwing is duplicated below. - if (get(id).isPresent()) - throw new IllegalArgumentException("An application with id '" + id + "' already exists"); - com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId.validate(id.application().value()); Optional<Tenant> tenant = controller.tenants().tenant(new TenantId(id.tenant().value())); @@ -266,7 +268,7 @@ public class ApplicationController { /** Deploys an application. If the application does not exist it is created. */ // TODO: Get rid of the options arg - public ActivateResult deployApplication(ApplicationId applicationId, Zone zone, + public ActivateResult deployApplication(ApplicationId applicationId, ZoneId zone, ApplicationPackage applicationPackage, DeployOptions options) { try (Lock lock = lock(applicationId)) { // TODO: Shouldn't this go through the above method? Seems you can cheat the checks here ... ? @@ -328,15 +330,35 @@ public class ApplicationController { throw new IllegalArgumentException("Rejecting deployment of " + application + " to " + zone + " as the requested version " + version + " is older than" + " the current version " + existingDeployment.version()); - } + } + + Optional<Rotation> rotation; + try (RotationLock rotationLock = rotationRepository.lock()) { + rotation = getRotation(application, zone, rotationLock); + if (rotation.isPresent()) { + application = application.with(rotation.get().id()); + store(application); // store assigned rotation even if deployment fails + registerRotationInDns(rotation.get(), application.rotation().get().dnsName()); + } + } + + // TODO: Improve config server client interface and simplify + Set<String> cnames = application.rotation() + .map(ApplicationRotation::dnsName) + .map(Collections::singleton) + .orElseGet(Collections::emptySet); + + Set<com.yahoo.vespa.hosted.controller.api.rotation.Rotation> rotations = rotation + .map(r -> new com.yahoo.vespa.hosted.controller.api.rotation.Rotation( + new com.yahoo.vespa.hosted.controller.api.identifiers.RotationId( + r.id().asString()), r.name())) + .map(Collections::singleton) + .orElseGet(Collections::emptySet); // Carry out deployment - DeploymentId deploymentId = new DeploymentId(applicationId, zone); - ApplicationRotation rotationInDns = registerRotationInDns(deploymentId, getOrAssignRotation(deploymentId, - applicationPackage)); - options = withVersion(version, options); + options = withVersion(version, options); ConfigServerClient.PreparedApplication preparedApplication = - configserverClient.prepare(deploymentId, options, rotationInDns.cnames(), rotationInDns.rotations(), + configserverClient.prepare(new DeploymentId(applicationId, zone), options, cnames, rotations, applicationPackage.zippedContent()); preparedApplication.activate(); application = application.withNewDeployment(zone, revision, version, clock.instant()); @@ -347,7 +369,7 @@ public class ApplicationController { } } - private ActivateResult unexpectedDeployment(ApplicationId applicationId, Zone zone, ApplicationPackage applicationPackage) { + private ActivateResult unexpectedDeployment(ApplicationId applicationId, ZoneId zone, ApplicationPackage applicationPackage) { Log logEntry = new Log(); logEntry.level = "WARNING"; logEntry.time = clock.instant().toEpochMilli(); @@ -384,9 +406,9 @@ public class ApplicationController { private LockedApplication deleteUnreferencedDeploymentJobs(LockedApplication application) { for (DeploymentJobs.JobType job : application.deploymentJobs().jobStatus().keySet()) { - Optional<Zone> zone = job.zone(controller.system()); + Optional<ZoneId> zone = job.zone(controller.system()); - if ( ! job.isProduction() || (zone.isPresent() && application.deploymentSpec().includes(zone.get().environment(), zone.map(Zone::region)))) + if ( ! job.isProduction() || (zone.isPresent() && application.deploymentSpec().includes(zone.get().environment(), zone.map(ZoneId::region)))) continue; application = application.withoutDeploymentJob(job); } @@ -430,35 +452,35 @@ public class ApplicationController { gitRevision.commit.id())); } - private ApplicationRotation registerRotationInDns(DeploymentId deploymentId, ApplicationRotation applicationRotation) { - ApplicationAlias alias = new ApplicationAlias(deploymentId.applicationId()); - if (applicationRotation.rotations().isEmpty()) return applicationRotation; - - Rotation rotation = applicationRotation.rotations().iterator().next(); // at this time there should be only one rotation assigned - String endpointName = alias.toString(); + /** Register a DNS name for rotation */ + private void registerRotationInDns(Rotation rotation, String dnsName) { try { - Optional<Record> record = nameService.findRecord(Record.Type.CNAME, endpointName); - if (!record.isPresent()) { - RecordId recordId = nameService.createCname(endpointName, rotation.rotationName); - log.info("Registered mapping with record ID " + recordId.id() + ": " + - endpointName + " -> " + rotation.rotationName); + Optional<Record> record = nameService.findRecord(Record.Type.CNAME, RecordName.from(dnsName)); + RecordData rotationName = RecordData.fqdn(rotation.name()); + if (record.isPresent()) { + // Ensure that the existing record points to the correct rotation + if (!record.get().data().equals(rotationName)) { + nameService.updateRecord(record.get().id(), rotationName); + log.info("Updated mapping for record ID " + record.get().id().asString() + ": '" + dnsName + + "' -> '" + rotation.name() + "'"); + } + } else { + RecordId id = nameService.createCname(RecordName.from(dnsName), rotationName); + log.info("Registered mapping with record ID " + id.asString() + ": '" + dnsName + "' -> '" + + rotation.name() + "'"); } - } - catch (RuntimeException e) { + } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to register CNAME", e); } - return new ApplicationRotation(Collections.singleton(endpointName), Collections.singleton(rotation)); } - private ApplicationRotation getOrAssignRotation(DeploymentId deploymentId, ApplicationPackage applicationPackage) { - if (deploymentId.zone().environment().equals(Environment.prod)) { - return new ApplicationRotation(Collections.emptySet(), - rotationRepository.getOrAssignRotation(deploymentId.applicationId(), - applicationPackage.deploymentSpec())); - } else { - return new ApplicationRotation(Collections.emptySet(), - Collections.emptySet()); + /** Get an available rotation, if deploying to a production zone and a service ID is specified */ + private Optional<Rotation> getRotation(Application application, ZoneId zone, RotationLock lock) { + if (zone.environment() != Environment.prod || + !application.deploymentSpec().globalServiceId().isPresent()) { + return Optional.empty(); } + return Optional.of(rotationRepository.getRotation(application, lock)); } /** Returns the endpoints of the deployment, or empty if obtaining them failed */ @@ -476,7 +498,8 @@ public class ApplicationController { return Optional.of(new InstanceEndpoints(endPointUrls)); } catch (RuntimeException e) { - log.log(Level.FINE, "Failed to get endpoint information for " + deploymentId, e); + log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId + ": " + + Exceptions.toMessageString(e)); return Optional.empty(); } } @@ -491,7 +514,7 @@ public class ApplicationController { if ( ! controller.applications().get(id).isPresent()) throw new NotExistsException("Could not delete application '" + id + "': Application not found"); - lockedOrThrow(id, application -> { + lockOrThrow(id, application -> { if ( ! application.deployments().isEmpty()) throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments"); @@ -521,24 +544,25 @@ public class ApplicationController { /** * Acquire a locked application to modify and store, if there is an application with the given id. * - * @param applicationId Id of the application to lock and get. - * @param actions Things to do with the locked application. + * @param applicationId ID of the application to lock and get. + * @param action Function which acts on the locked application. */ - public void lockedIfPresent(ApplicationId applicationId, Consumer<LockedApplication> actions) { + public void lockIfPresent(ApplicationId applicationId, Consumer<LockedApplication> action) { try (Lock lock = lock(applicationId)) { - get(applicationId).map(application -> new LockedApplication(application, lock)).ifPresent(actions); + get(applicationId).map(application -> new LockedApplication(application, lock)).ifPresent(action); } } /** * Acquire a locked application to modify and store, or throw an exception if no application has the given id. * - * @param applicationId Id of the application to lock and require. - * @param actions Things to do with the locked application. + * @param applicationId ID of the application to lock and require. + * @param action Function which acts on the locked application. + * @throws IllegalArgumentException when application does not exist. */ - public void lockedOrThrow(ApplicationId applicationId, Consumer<LockedApplication> actions) { + public void lockOrThrow(ApplicationId applicationId, Consumer<LockedApplication> action) { try (Lock lock = lock(applicationId)) { - actions.accept(new LockedApplication(require(applicationId), lock)); + action.accept(new LockedApplication(require(applicationId), lock)); } } @@ -551,18 +575,14 @@ public class ApplicationController { deploymentTrigger.triggerFromCompletion(report); } - // TODO: Collapse this method and the next - public void restart(DeploymentId deploymentId) { - try { - configserverClient.restart(deploymentId, Optional.empty()); - } - catch (NoInstanceException e) { - throw new IllegalArgumentException("Could not restart " + deploymentId + ": No such deployment"); - } - } - public void restartHost(DeploymentId deploymentId, Hostname hostname) { + /** + * Tells config server to schedule a restart of all nodes in this deployment + * + * @param hostname If non-empty, restart will only be scheduled for this host + */ + public void restart(DeploymentId deploymentId, Optional<Hostname> hostname) { try { - configserverClient.restart(deploymentId, Optional.of(hostname)); + configserverClient.restart(deploymentId, hostname); } catch (NoInstanceException e) { throw new IllegalArgumentException("Could not restart " + deploymentId + ": No such deployment"); @@ -570,7 +590,7 @@ public class ApplicationController { } /** Deactivate application in the given zone */ - public void deactivate(Application application, Zone zone) { + public void deactivate(Application application, ZoneId zone) { deactivate(application, zone, Optional.empty(), false); } @@ -579,13 +599,13 @@ public class ApplicationController { deactivate(application, deployment.zone(), Optional.of(deployment), requireThatDeploymentHasExpired); } - private void deactivate(Application application, Zone zone, Optional<Deployment> deployment, + private void deactivate(Application application, ZoneId zone, Optional<Deployment> deployment, boolean requireThatDeploymentHasExpired) { if (requireThatDeploymentHasExpired && deployment.isPresent() && ! DeploymentExpirer.hasExpired(controller.zoneRegistry(), deployment.get(), clock.instant())) return; - lockedOrThrow(application.id(), lockedApplication -> + lockOrThrow(application.id(), lockedApplication -> store(deactivate(lockedApplication, zone))); } @@ -594,7 +614,7 @@ public class ApplicationController { * * @return the application with the deployment in the given zone removed */ - private LockedApplication deactivate(LockedApplication application, Zone zone) { + private LockedApplication deactivate(LockedApplication application, ZoneId zone) { try { configserverClient.deactivate(new DeploymentId(application.id(), zone)); } @@ -624,7 +644,7 @@ public class ApplicationController { } /** Returns whether a direct deployment to given zone is allowed */ - private static boolean canDeployDirectlyTo(Zone zone, DeployOptions options) { + private static boolean canDeployDirectlyTo(ZoneId zone, DeployOptions options) { return ! options.screwdriverBuildJob.isPresent() || options.screwdriverBuildJob.get().screwdriverId == null || zone.environment().isManuallyDeployed(); @@ -640,20 +660,8 @@ public class ApplicationController { }); } - - private static final class ApplicationRotation { - - private final ImmutableSet<String> cnames; - private final ImmutableSet<Rotation> rotations; - - public ApplicationRotation(Set<String> cnames, Set<Rotation> rotations) { - this.cnames = ImmutableSet.copyOf(cnames); - this.rotations = ImmutableSet.copyOf(rotations); - } - - public Set<String> cnames() { return cnames; } - public Set<Rotation> rotations() { return rotations; } - + public RotationRepository rotationRepository() { + return rotationRepository; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index b854ad3f771..71a0a7f6297 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -6,7 +6,6 @@ import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.component.Vtag; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; @@ -25,12 +24,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingSe import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import com.yahoo.vespa.hosted.rotation.RotationRepository; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.net.URI; @@ -39,7 +38,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.logging.Logger; /** @@ -64,7 +62,6 @@ public class Controller extends AbstractComponent { private final ApplicationController applicationController; private final TenantController tenantController; private final Clock clock; - private final RotationRepository rotationRepository; private final GitHub gitHub; private final EntityService entityService; private final GlobalRoutingService globalRoutingService; @@ -82,19 +79,19 @@ public class Controller extends AbstractComponent { * @param curator the curator instance storing working state shared between controller instances */ @Inject - public Controller(ControllerDb db, CuratorDb curator, RotationRepository rotationRepository, + public Controller(ControllerDb db, CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, EntityService entityService, Organization organization, GlobalRoutingService globalRoutingService, ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, MetricsService metricsService, NameService nameService, RoutingGenerator routingGenerator, Chef chefClient, AthenzClientFactory athenzClientFactory) { - this(db, curator, rotationRepository, + this(db, curator, rotationsConfig, gitHub, entityService, organization, globalRoutingService, zoneRegistry, configServerClient, metricsService, nameService, routingGenerator, chefClient, Clock.systemUTC(), athenzClientFactory); } - public Controller(ControllerDb db, CuratorDb curator, RotationRepository rotationRepository, + public Controller(ControllerDb db, CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, EntityService entityService, Organization organization, GlobalRoutingService globalRoutingService, ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, @@ -103,7 +100,7 @@ public class Controller extends AbstractComponent { AthenzClientFactory athenzClientFactory) { Objects.requireNonNull(db, "Controller db cannot be null"); Objects.requireNonNull(curator, "Curator cannot be null"); - Objects.requireNonNull(rotationRepository, "Rotation repository cannot be null"); + Objects.requireNonNull(rotationsConfig, "RotationsConfig cannot be null"); Objects.requireNonNull(gitHub, "GitHubClient cannot be null"); Objects.requireNonNull(entityService, "EntityService cannot be null"); Objects.requireNonNull(organization, "Organization cannot be null"); @@ -117,7 +114,6 @@ public class Controller extends AbstractComponent { Objects.requireNonNull(clock, "Clock cannot be null"); Objects.requireNonNull(athenzClientFactory, "Athens cannot be null"); - this.rotationRepository = rotationRepository; this.curator = curator; this.gitHub = gitHub; this.entityService = entityService; @@ -130,7 +126,8 @@ public class Controller extends AbstractComponent { this.clock = clock; this.athenzClientFactory = athenzClientFactory; - applicationController = new ApplicationController(this, db, curator, rotationRepository, athenzClientFactory, + applicationController = new ApplicationController(this, db, curator, athenzClientFactory, + rotationsConfig, nameService, configServerClient, routingGenerator, clock); tenantController = new TenantController(this, db, curator, entityService, athenzClientFactory); } @@ -181,10 +178,6 @@ public class Controller extends AbstractComponent { return kibanaHost.map(uri -> uri.resolve(kibanaPath)).orElse(null); } - public Set<URI> getRotationUris(ApplicationId id) { - return rotationRepository.getRotationUris(id); - } - public Map<String, RotationStatus> getHealthStatus(String hostname) { return globalRoutingService.getHealthStatus(hostname); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index e8c8f8a389c..72ed1a42435 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -6,10 +6,12 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; +import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -18,7 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; - +import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.LinkedHashMap; @@ -34,12 +36,6 @@ import java.util.Optional; */ public class LockedApplication extends Application { - private LockedApplication(Builder builder) { - super(builder.applicationId, builder.deploymentSpec, builder.validationOverrides, - builder.deployments, builder.deploymentJobs, builder.deploying, - builder.hasOutstandingChange, builder.ownershipIssueId, builder.metrics); - } - /** * Used to create a locked application * @@ -50,6 +46,12 @@ public class LockedApplication extends Application { this(new Builder(application)); } + private LockedApplication(Builder builder) { + super(builder.applicationId, builder.deploymentSpec, builder.validationOverrides, + builder.deployments, builder.deploymentJobs, builder.deploying, + builder.hasOutstandingChange, builder.ownershipIssueId, builder.metrics, builder.rotation); + } + public LockedApplication withProjectId(long projectId) { return new LockedApplication(new Builder(this).with(deploymentJobs().withProjectId(projectId))); } @@ -67,7 +69,7 @@ public class LockedApplication extends Application { return new LockedApplication(new Builder(this).with(deploymentJobs().withTriggering(type, change, version, revision, reason, triggerTime))); } - public LockedApplication withNewDeployment(Zone zone, ApplicationRevision revision, Version version, Instant instant) { + public LockedApplication withNewDeployment(ZoneId zone, ApplicationRevision revision, Version version, Instant instant) { // Use info from previous deployment if available, otherwise create a new one. Deployment previousDeployment = deployments().getOrDefault(zone, new Deployment(zone, revision, version, instant)); Deployment newDeployment = new Deployment(zone, revision, version, instant, @@ -77,27 +79,27 @@ public class LockedApplication extends Application { return with(newDeployment); } - public LockedApplication withClusterUtilization(Zone zone, Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization) { + public LockedApplication withClusterUtilization(ZoneId zone, Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization) { Deployment deployment = deployments().get(zone); if (deployment == null) return this; // No longer deployed in this zone. return with(deployment.withClusterUtils(clusterUtilization)); } - public LockedApplication withClusterInfo(Zone zone, Map<ClusterSpec.Id, ClusterInfo> clusterInfo) { + public LockedApplication withClusterInfo(ZoneId zone, Map<ClusterSpec.Id, ClusterInfo> clusterInfo) { Deployment deployment = deployments().get(zone); if (deployment == null) return this; // No longer deployed in this zone. return with(deployment.withClusterInfo(clusterInfo)); } - public LockedApplication with(Zone zone, DeploymentMetrics deploymentMetrics) { + public LockedApplication with(ZoneId zone, DeploymentMetrics deploymentMetrics) { Deployment deployment = deployments().get(zone); if (deployment == null) return this; // No longer deployed in this zone. return with(deployment.withMetrics(deploymentMetrics)); } - public LockedApplication withoutDeploymentIn(Zone zone) { - Map<Zone, Deployment> deployments = new LinkedHashMap<>(deployments()); + public LockedApplication withoutDeploymentIn(ZoneId zone) { + Map<ZoneId, Deployment> deployments = new LinkedHashMap<>(deployments()); deployments.remove(zone); return new LockedApplication(new Builder(this).with(deployments)); } @@ -130,6 +132,10 @@ public class LockedApplication extends Application { return new LockedApplication(new Builder(this).with(metrics)); } + public LockedApplication with(RotationId rotation) { + return new LockedApplication(new Builder(this).with(rotation)); + } + public Version deployVersionFor(DeploymentJobs.JobType jobType, Controller controller) { return jobType == JobType.component ? controller.systemVersion() @@ -144,7 +150,7 @@ public class LockedApplication extends Application { /** Don't expose non-leaf sub-objects. */ private LockedApplication with(Deployment deployment) { - Map<Zone, Deployment> deployments = new LinkedHashMap<>(deployments()); + Map<ZoneId, Deployment> deployments = new LinkedHashMap<>(deployments()); deployments.put(deployment.zone(), deployment); return new LockedApplication(new Builder(this).with(deployments)); } @@ -155,12 +161,13 @@ public class LockedApplication extends Application { private final ApplicationId applicationId; private DeploymentSpec deploymentSpec; private ValidationOverrides validationOverrides; - private Map<Zone, Deployment> deployments; + private Map<ZoneId, Deployment> deployments; private DeploymentJobs deploymentJobs; private Optional<Change> deploying; private boolean hasOutstandingChange; private Optional<IssueId> ownershipIssueId; private ApplicationMetrics metrics; + private Optional<RotationId> rotation; private Builder(Application application) { this.applicationId = application.id(); @@ -172,16 +179,53 @@ public class LockedApplication extends Application { this.hasOutstandingChange = application.hasOutstandingChange(); this.ownershipIssueId = application.ownershipIssueId(); this.metrics = application.metrics(); + this.rotation = application.rotation().map(ApplicationRotation::id); + } + + private Builder with(DeploymentSpec deploymentSpec) { + this.deploymentSpec = deploymentSpec; + return this; } - private Builder with(DeploymentSpec deploymentSpec) { this.deploymentSpec = deploymentSpec; return this; } - private Builder with(ValidationOverrides validationOverrides) { this.validationOverrides = validationOverrides; return this; } - private Builder with(Map<Zone, Deployment> deployments) { this.deployments = deployments; return this; } - private Builder with(DeploymentJobs deploymentJobs) { this.deploymentJobs = deploymentJobs; return this; } - private Builder withDeploying(Optional<Change> deploying) { this.deploying = deploying; return this; } - private Builder with(boolean hasOutstandingChange) { this.hasOutstandingChange = hasOutstandingChange; return this; } - private Builder withOwnershipIssueId(Optional<IssueId> ownershipIssueId) { this.ownershipIssueId = ownershipIssueId; return this; } - private Builder with(ApplicationMetrics metrics) { this.metrics = metrics; return this; } + private Builder with(ValidationOverrides validationOverrides) { + this.validationOverrides = validationOverrides; + return this; + } + + private Builder with(Map<ZoneId, Deployment> deployments) { + this.deployments = deployments; + return this; + } + + private Builder with(DeploymentJobs deploymentJobs) { + this.deploymentJobs = deploymentJobs; + return this; + } + + private Builder withDeploying(Optional<Change> deploying) { + this.deploying = deploying; + return this; + } + + private Builder with(boolean hasOutstandingChange) { + this.hasOutstandingChange = hasOutstandingChange; + return this; + } + + private Builder withOwnershipIssueId(Optional<IssueId> ownershipIssueId) { + this.ownershipIssueId = ownershipIssueId; + return this; + } + + private Builder with(ApplicationMetrics metrics) { + this.metrics = metrics; + return this; + } + + private Builder with(RotationId rotation) { + this.rotation = Optional.of(rotation); + return this; + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index 9530e9a982c..a52098a4a0f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -12,11 +12,11 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.PersistenceException; @@ -67,7 +67,7 @@ public class TenantController { public List<Tenant> asList(UserId user) { Set<UserGroup> userGroups = entityService.getUserGroups(user); Set<AthenzDomain> userDomains = new HashSet<>(athenzClientFactory.createZtsClientWithServicePrincipal() - .getTenantDomainsForUser(AthenzUtils.createPrincipal(user))); + .getTenantDomainsForUser(AthenzUser.fromUserId(user))); Predicate<Tenant> hasUsersGroup = (tenant) -> tenant.getUserGroup().isPresent() && userGroups.contains(tenant.getUserGroup().get()); Predicate<Tenant> hasUsersDomain = (tenant) -> tenant.getAthensDomain().isPresent() && userDomains.contains(tenant.getAthensDomain().get()); @@ -200,8 +200,7 @@ public class TenantController { try (Lock lock = lock(tenantId)) { Tenant existing = tenant(tenantId).orElseThrow(() -> new NotExistsException(tenantId)); if (existing.isAthensTenant()) return existing; // nothing to do - log.info("Starting migration of " + existing + " to Athenz domain " + tenantDomain.id() + - " using " + nToken.getPrincipal()); + log.info("Starting migration of " + existing + " to Athenz domain " + tenantDomain.id()); if (tenantHaving(tenantDomain).isPresent()) throw new IllegalArgumentException("Could not migrate " + existing + " to " + tenantDomain + ": " + "This domain is already used by " + tenantHaving(tenantDomain).get()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java deleted file mode 100644 index a9e144a3227..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/api/ApplicationAlias.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api; - -import com.yahoo.config.provision.ApplicationId; - -import java.net.URI; -import java.net.URISyntaxException; - -/** - * A DNS alias for an application endpoint. - * - * @author smorgrav - */ -public class ApplicationAlias { - - private static final String dnsSuffix = "global.vespa.yahooapis.com"; - - private final ApplicationId applicationId; - - public ApplicationAlias(ApplicationId applicationId) { - this.applicationId = applicationId; - } - - @Override - public String toString() { - return String.format("%s.%s.%s", - toDns(applicationId.application().value()), - toDns(applicationId.tenant().value()), - dnsSuffix); - } - - private String toDns(String id) { - return id.replace('_', '-'); - } - - public URI toHttpUri() { - try { - return new URI("http://" + this + ":4080/"); - } catch(URISyntaxException use) { - throw new RuntimeException("Illegal URI syntax"); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ApplicationAlias that = (ApplicationAlias) o; - - return applicationId.equals(that.applicationId); - } - - @Override - public int hashCode() { return applicationId.hashCode(); } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRotation.java new file mode 100644 index 00000000000..e4aed04a01c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationRotation.java @@ -0,0 +1,51 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.rotation.RotationId; + +import java.net.URI; + +/** + * Represents an application's global rotation. + * + * @author mpolden + */ +public class ApplicationRotation { + + public static final String DNS_SUFFIX = "global.vespa.yahooapis.com"; + private static final int port = 4080; + + private final URI url; + private final RotationId id; + + public ApplicationRotation(ApplicationId application, RotationId id) { + this.url = URI.create(String.format("http://%s.%s.%s:%d/", + sanitize(application.application().value()), + sanitize(application.tenant().value()), + DNS_SUFFIX, + port)); + this.id = id; + } + + /** ID of the rotation */ + public RotationId id() { + return id; + } + + /** URL to this rotation */ + public URI url() { + return url; + } + + /** DNS name for this rotation */ + public String dnsName() { + return url.getHost(); + } + + /** Sanitize by translating '_' to '-' as the former is not allowed in a DNS name */ + private static String sanitize(String s) { + return s.replace('_', '-'); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 98ae5ed1762..b9d07249cb2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import java.time.Instant; import java.util.HashMap; @@ -18,7 +19,7 @@ import java.util.Objects; */ public class Deployment { - private final Zone zone; + private final ZoneId zone; private final ApplicationRevision revision; private final Version version; private final Instant deployTime; @@ -26,11 +27,11 @@ public class Deployment { private final Map<Id, ClusterInfo> clusterInfo; private final DeploymentMetrics metrics; - public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime) { + public Deployment(ZoneId zone, ApplicationRevision revision, Version version, Instant deployTime) { this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>(), new DeploymentMetrics()); } - public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime, + public Deployment(ZoneId zone, ApplicationRevision revision, Version version, Instant deployTime, Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo, DeploymentMetrics metrics) { Objects.requireNonNull(zone, "zone cannot be null"); Objects.requireNonNull(revision, "revision cannot be null"); @@ -49,7 +50,7 @@ public class Deployment { } /** Returns the zone this was deployed to */ - public Zone zone() { return zone; } + public ZoneId zone() { return zone; } /** Returns the revision of the application which was deployed */ public ApplicationRevision revision() { return revision; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java index 98f8c2a3d99..ec8b2d6d019 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java @@ -7,19 +7,17 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import java.time.Instant; import java.util.Collection; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -161,34 +159,36 @@ public class DeploymentJobs { /** Job types that exist in the build system */ public enum JobType { - - component ("component" ), - systemTest ("system-test" , zone("test" , "us-east-1" ), zone(SystemName.cd, "test" , "cd-us-central-1")), - stagingTest ("staging-test" , zone("staging", "us-east-3" ), zone(SystemName.cd, "staging", "cd-us-central-1")), - productionCorpUsEast1 ("production-corp-us-east-1" , zone("prod" , "corp-us-east-1")), - productionUsEast3 ("production-us-east-3" , zone("prod" , "us-east-3" )), - productionUsWest1 ("production-us-west-1" , zone("prod" , "us-west-1" )), - productionUsCentral1 ("production-us-central-1" , zone("prod" , "us-central-1" )), - productionApNortheast1 ("production-ap-northeast-1" , zone("prod" , "ap-northeast-1")), - productionApNortheast2 ("production-ap-northeast-2" , zone("prod" , "ap-northeast-2")), - productionApSoutheast1 ("production-ap-southeast-1" , zone("prod" , "ap-southeast-1")), - productionEuWest1 ("production-eu-west-1" , zone("prod" , "eu-west-1" )), - productionCdUsCentral1 ("production-cd-us-central-1", zone(SystemName.cd, "prod", "cd-us-central-1")), - productionCdUsCentral2 ("production-cd-us-central-2", zone(SystemName.cd, "prod", "cd-us-central-2")); +// | enum name ------------| job name ------------------| Zone in main system ---------------------------------------| Zone in CD system ------------------------------------------- + component ("component" , null , null ), + systemTest ("system-test" , ZoneId.from("test" , "us-east-1") , ZoneId.from("test" , "cd-us-central-1")), + stagingTest ("staging-test" , ZoneId.from("staging", "us-east-3") , ZoneId.from("staging", "cd-us-central-1")), + productionCorpUsEast1 ("production-corp-us-east-1" , ZoneId.from("prod" , "corp-us-east-1") , null ), + productionUsEast3 ("production-us-east-3" , ZoneId.from("prod" , "us-east-3") , null ), + productionUsWest1 ("production-us-west-1" , ZoneId.from("prod" , "us-west-1") , null ), + productionUsCentral1 ("production-us-central-1" , ZoneId.from("prod" , "us-central-1") , null ), + productionApNortheast1 ("production-ap-northeast-1" , ZoneId.from("prod" , "ap-northeast-1") , null ), + productionApNortheast2 ("production-ap-northeast-2" , ZoneId.from("prod" , "ap-northeast-2") , null ), + productionApSoutheast1 ("production-ap-southeast-1" , ZoneId.from("prod" , "ap-southeast-1") , null ), + productionEuWest1 ("production-eu-west-1" , ZoneId.from("prod" , "eu-west-1") , null ), + productionCdUsCentral1 ("production-cd-us-central-1", null , ZoneId.from("prod" , "cd-us-central-1")), + productionCdUsCentral2 ("production-cd-us-central-2", null , ZoneId.from("prod" , "cd-us-central-2")); private final String jobName; - private final ImmutableMap<SystemName, Zone> zones; + private final ImmutableMap<SystemName, ZoneId> zones; - JobType(String jobName, Zone... zones) { + JobType(String jobName, ZoneId mainZone, ZoneId cdZone) { this.jobName = jobName; - this.zones = ImmutableMap.copyOf(Stream.of(zones).collect(Collectors.toMap(zone -> zone.system(), - zone -> zone))); + ImmutableMap.Builder<SystemName, ZoneId> builder = ImmutableMap.builder(); + if (mainZone != null) builder.put(SystemName.main, mainZone); + if (cdZone != null) builder.put(SystemName.cd, cdZone); + this.zones = builder.build(); } public String jobName() { return jobName; } /** Returns the zone for this job in the given system, or empty if this job does not have a zone */ - public Optional<Zone> zone(SystemName system) { + public Optional<ZoneId> zone(SystemName system) { return Optional.ofNullable(zones.get(system)); } @@ -207,7 +207,7 @@ public class DeploymentJobs { /** Returns the region of this job type, or null if it does not have a region */ public Optional<RegionName> region(SystemName system) { - return zone(system).map(Zone::region); + return zone(system).map(ZoneId::region); } public static JobType fromJobName(String jobName) { @@ -217,7 +217,7 @@ public class DeploymentJobs { } /** Returns the job type for the given zone */ - public static Optional<JobType> from(SystemName system, Zone zone) { + public static Optional<JobType> from(SystemName system, ZoneId zone) { return Stream.of(values()) .filter(job -> job.zone(system).filter(zone::equals).isPresent()) .findAny(); @@ -229,16 +229,9 @@ public class DeploymentJobs { case test: return Optional.of(systemTest); case staging: return Optional.of(stagingTest); } - return from(system, new Zone(environment, region)); - } - - private static Zone zone(SystemName system, String environment, String region) { - return new Zone(system, Environment.from(environment), RegionName.from(region)); + return from(system, ZoneId.from(environment, region)); } - private static Zone zone(String environment, String region) { - return new Zone(Environment.from(environment), RegionName.from(region)); - } } /** A job report. This class is immutable. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java deleted file mode 100644 index 8614414dc95..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -/** - * @author bjorncs - */ -public enum ApplicationAction { - deploy("deployer"), - read("reader"), - write("writer"); - - public final String roleName; - - ApplicationAction(String roleName) { - this.roleName = roleName; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzClientFactory.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzClientFactory.java deleted file mode 100644 index b6a21f94f74..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzClientFactory.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -/** - * @author bjorncs - */ -public interface AthenzClientFactory { - - ZmsClient createZmsClientWithServicePrincipal(); - - ZtsClient createZtsClientWithServicePrincipal(); - - ZmsClient createZmsClientWithAuthorizedServiceToken(NToken authorizedServiceToken); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPrincipal.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPrincipal.java deleted file mode 100644 index 1e4952a39c5..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPrincipal.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; - -import java.security.Principal; -import java.util.Objects; - -/** - * @author bjorncs - */ -public class AthenzPrincipal implements Principal { - - private final AthenzDomain domain; - private final UserId userId; - - public AthenzPrincipal(AthenzDomain domain, UserId userId) { - this.domain = domain; - this.userId = userId; - } - - public UserId getUserId() { - return userId; - } - - public AthenzDomain getDomain() { - return domain; - } - - public String toYRN() { - return domain.id() + "." + userId.id(); - } - - @Override - public String getName() { - return userId.id(); - } - - @Override - public String toString() { - return "AthenzPrincipal{" + - "domain=" + domain + - ", userId=" + userId + - '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AthenzPrincipal that = (AthenzPrincipal) o; - return Objects.equals(domain, that.domain) && - Objects.equals(userId, that.userId); - } - - @Override - public int hashCode() { - return Objects.hash(domain, userId); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPublicKey.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPublicKey.java deleted file mode 100644 index 01596ead0f4..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPublicKey.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import java.security.PublicKey; -import java.util.Objects; - -/** - * @author bjorncs - */ -public class AthenzPublicKey { - - private final PublicKey publicKey; - private final String keyId; - - public AthenzPublicKey(PublicKey publicKey, String keyId) { - this.publicKey = publicKey; - this.keyId = keyId; - } - - public PublicKey getPublicKey() { - return publicKey; - } - - public String getKeyId() { - return keyId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AthenzPublicKey that = (AthenzPublicKey) o; - return Objects.equals(publicKey, that.publicKey) && - Objects.equals(keyId, that.keyId); - } - - @Override - public int hashCode() { - return Objects.hash(publicKey, keyId); - } - - @Override - public String toString() { - return "AthenzPublicKey{" + - "publicKey=" + publicKey + - ", keyId='" + keyId + '\'' + - '}'; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzService.java deleted file mode 100644 index 37c6459b687..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzService.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; - -import java.util.Objects; - -/** - * @author bjorncs - */ -public class AthenzService { - - private final AthenzDomain domain; - private final String serviceName; - - public AthenzService(AthenzDomain domain, String serviceName) { - this.domain = domain; - this.serviceName = serviceName; - } - - public AthenzService(String domain, String serviceName) { - this(new AthenzDomain(domain), serviceName); - } - - public String toFullServiceName() { - return domain.id() + "." + serviceName; - } - - public AthenzDomain getDomain() { - return domain; - } - - public String getServiceName() { - return serviceName; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AthenzService that = (AthenzService) o; - return Objects.equals(domain, that.domain) && - Objects.equals(serviceName, that.serviceName); - } - - @Override - public int hashCode() { - return Objects.hash(domain, serviceName); - } - - @Override - public String toString() { - return String.format("AthenzService(%s)", toFullServiceName()); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUtils.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUtils.java deleted file mode 100644 index 18bd626369d..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUtils.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; - -/** - * @author bjorncs - */ -public class AthenzUtils { - - private AthenzUtils() {} - - public static final AthenzDomain USER_PRINCIPAL_DOMAIN = new AthenzDomain("user"); - public static final AthenzDomain SCREWDRIVER_DOMAIN = new AthenzDomain("cd.screwdriver.project"); - public static final AthenzService ZMS_ATHENZ_SERVICE = new AthenzService("sys.auth", "zms"); - - public static AthenzPrincipal createPrincipal(UserId userId) { - return new AthenzPrincipal(USER_PRINCIPAL_DOMAIN, userId); - } - - public static AthenzPrincipal createPrincipal(ScrewdriverId screwdriverId) { - return new AthenzPrincipal(SCREWDRIVER_DOMAIN, new UserId("sd" + screwdriverId.id())); - } - - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/InvalidTokenException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/InvalidTokenException.java deleted file mode 100644 index e41bd8d4283..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/InvalidTokenException.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -/** - * @author bjorncs - */ -public class InvalidTokenException extends Exception { - public InvalidTokenException(String message) { - super(message); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/NToken.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/NToken.java deleted file mode 100644 index 7e3abeb77d9..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/NToken.java +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.athenz.auth.token.PrincipalToken; -import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; - -import java.security.PrivateKey; -import java.security.PublicKey; -import java.util.Objects; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.concurrent.TimeUnit; - -/** - * Represents an Athenz NToken (principal token) - * - * @author bjorncs - */ -// TODO Split out encoding/decoding of token into separate class. Move NToken to controller-api. -public class NToken { - - // Max allowed skew in token timestamp (only for creation, not expiry timestamp) - private static final int ALLOWED_TIMESTAMP_OFFSET = (int) TimeUnit.SECONDS.toSeconds(300); - - private final PrincipalToken token; - - // Note: PrincipalToken does not provide any way of constructing an instance from a unsigned token string - public NToken(String signedToken) { - try { - this.token = new PrincipalToken(signedToken); - if (this.token.getSignature() == null) { - throw new IllegalArgumentException("Signature missing (unsigned token)"); - } - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Malformed NToken: " + e.getMessage()); - } - } - - public AthenzPrincipal getPrincipal() { - return new AthenzPrincipal(getDomain(), getUser()); - } - - public UserId getUser() { - return new UserId(token.getName()); - } - - public AthenzDomain getDomain() { - return new AthenzDomain(token.getDomain()); - } - - public String getToken() { - return token.getSignedToken(); - } - - public String getKeyId() { - return token.getKeyId(); - } - - public void validateSignatureAndExpiration(PublicKey publicKey) throws InvalidTokenException { - StringBuilder errorMessageBuilder = new StringBuilder(); - if (!token.validate(publicKey, ALLOWED_TIMESTAMP_OFFSET, true, errorMessageBuilder)) { - throw new InvalidTokenException("NToken is expired or has invalid signature: " + errorMessageBuilder.toString()); - } - } - - @Override - public String toString() { - return String.format("NToken(%s)", getToken()); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - NToken nToken = (NToken) o; - return Objects.equals(getToken(), nToken.getToken()); // PrincipalToken does not implement equals() - } - - @Override - public int hashCode() { - return Objects.hash(getToken()); // PrincipalToken does not implement hashcode() - } - - public static class Builder { - - private final String version; - private final AthenzPrincipal principal; - private final PrivateKey privateKey; - private final String keyId; - private Optional<String> salt = Optional.empty(); - private Optional<String> hostname = Optional.empty(); - private Optional<String> ip = Optional.empty(); - private OptionalLong issueTime = OptionalLong.empty(); - private OptionalLong expirationWindow = OptionalLong.empty(); - - /** - * NOTE: We must have some signature, else we might end up with problems later on as - * {@link PrincipalToken#PrincipalToken(String)} only accepts signed token - * (supplying an unsigned token to the constructor will result in inconsistent state) - */ - public Builder(String version, AthenzPrincipal principal, PrivateKey privateKey, String keyId) { - this.version = version; - this.principal = principal; - this.privateKey = privateKey; - this.keyId = keyId; - } - - public Builder salt(String salt) { - this.salt = Optional.of(salt); - return this; - } - - public Builder hostname(String hostname) { - this.hostname = Optional.of(hostname); - return this; - } - - public Builder ip(String ip) { - this.ip = Optional.of(ip); - return this; - } - - public Builder issueTime(long issueTime) { - this.issueTime = OptionalLong.of(issueTime); - return this; - } - - public Builder expirationWindow(long expirationWindow) { - this.expirationWindow = OptionalLong.of(expirationWindow); - return this; - } - - public NToken build() { - PrincipalToken token = new PrincipalToken.Builder(version, principal.getDomain().id(), principal.getName()) - .keyId(this.keyId) - .salt(this.salt.orElse(null)) - .host(this.hostname.orElse(null)) - .ip(this.ip.orElse(null)) - .issueTime(this.issueTime.orElse(0)) - .expirationWindow(this.expirationWindow.orElse(0)) - .build(); - token.sign(this.privateKey); - return new NToken(token.getSignedToken()); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsClient.java deleted file mode 100644 index 407bce05c6e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsClient.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; - -import java.util.List; - -/** - * @author bjorncs - */ -public interface ZmsClient { - - void createTenant(AthenzDomain tenantDomain); - - void deleteTenant(AthenzDomain tenantDomain); - - void addApplication(AthenzDomain tenantDomain, ApplicationId applicationName); - - void deleteApplication(AthenzDomain tenantDomain, ApplicationId applicationName); - - boolean hasApplicationAccess(AthenzPrincipal principal, ApplicationAction action, AthenzDomain tenantDomain, ApplicationId applicationName); - - boolean hasTenantAdminAccess(AthenzPrincipal principal, AthenzDomain tenantDomain); - - // Used before vespa tenancy is established for the domain. - boolean isDomainAdmin(AthenzPrincipal principal, AthenzDomain domain); - - List<AthenzDomain> getDomainList(String prefix); - - AthenzPublicKey getPublicKey(AthenzService service, String keyId); - - List<AthenzPublicKey> getPublicKeys(AthenzService service); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsException.java deleted file mode 100644 index 59548339d11..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsException.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.athenz.zms.ZMSClientException; - -/** - * @author bjorncs - */ -public class ZmsException extends RuntimeException { - - private final int code; - - public ZmsException(ZMSClientException e) { - super(e.getMessage(), e); - this.code = e.getCode(); - } - - - public int getCode() { - return code; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsKeystore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsKeystore.java deleted file mode 100644 index 93fed95c768..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsKeystore.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import java.security.PublicKey; -import java.util.Optional; - -/** - * @author bjorncs - */ -public interface ZmsKeystore { - - Optional<PublicKey> getPublicKey(AthenzService service, String keyId); - - default void preloadKeys(AthenzService service) { /* Default implementation is noop */ } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsClient.java deleted file mode 100644 index f400ba2eb99..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsClient.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; - -import java.util.List; - -/** - * @author bjorncs - */ -public interface ZtsClient { - - List<AthenzDomain> getTenantDomainsForUser(AthenzPrincipal principal); - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsException.java deleted file mode 100644 index cb0b21ba459..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsException.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.athenz; - -import com.yahoo.athenz.zts.ZTSClientException; - -/** - * @author bjorncs - */ -public class ZtsException extends RuntimeException { - - private final int code; - - public ZtsException(ZTSClientException e) { - super(e.getMessage(), e); - this.code = e.getCode(); - } - - - public int getCode() { - return code; - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java index 51865be04fa..328461355db 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java @@ -6,10 +6,10 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import java.util.concurrent.Executor; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTrustStoreConfigurator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTrustStoreConfigurator.java new file mode 100644 index 00000000000..939a5667a36 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTrustStoreConfigurator.java @@ -0,0 +1,45 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.filter; + +import com.google.inject.Inject; +import com.yahoo.jdisc.http.ssl.SslTrustStoreConfigurator; +import com.yahoo.jdisc.http.ssl.SslTrustStoreContext; +import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; + +/** + * Load trust store with Athenz CA certificates + * + * @author bjorncs + */ +public class AthenzTrustStoreConfigurator implements SslTrustStoreConfigurator { + + private final KeyStore trustStore; + + @Inject + public AthenzTrustStoreConfigurator(AthenzConfig config) { + this.trustStore = createTrustStore(new File(config.athenzCaTrustStore())); + } + + private static KeyStore createTrustStore(File trustStoreFile) { + try (FileInputStream in = new FileInputStream(trustStoreFile)) { + KeyStore trustStore = KeyStore.getInstance("JKS"); + trustStore.load(in, "changeit".toCharArray()); + return trustStore; + } catch (IOException | CertificateException | NoSuchAlgorithmException | KeyStoreException e) { + throw new RuntimeException(e); + } + } + + @Override + public void configure(SslTrustStoreContext context) { + context.updateTrustStore(trustStore); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java index f43d2d8e80e..69f59ebabe2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java @@ -1,17 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.filter; +import com.yahoo.athenz.auth.token.PrincipalToken; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import java.security.PublicKey; +import java.time.Duration; import java.util.Optional; import java.util.logging.Logger; -import static com.yahoo.vespa.hosted.controller.athenz.AthenzUtils.ZMS_ATHENZ_SERVICE; +import static com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils.ZMS_ATHENZ_SERVICE; /** * Validates the content of an NToken: @@ -22,6 +26,9 @@ import static com.yahoo.vespa.hosted.controller.athenz.AthenzUtils.ZMS_ATHENZ_SE */ class NTokenValidator { + // Max allowed skew in token timestamp (only for creation, not expiry timestamp) + private static final long ALLOWED_TIMESTAMP_OFFSET = Duration.ofMinutes(5).getSeconds(); + private static final Logger log = Logger.getLogger(NTokenValidator.class.getName()); private final ZmsKeystore keystore; @@ -35,10 +42,15 @@ class NTokenValidator { } AthenzPrincipal validate(NToken token) throws InvalidTokenException { - PublicKey zmsPublicKey = getPublicKey(token.getKeyId()) + PrincipalToken principalToken = new PrincipalToken(token.getRawToken()); + PublicKey zmsPublicKey = getPublicKey(principalToken.getKeyId()) .orElseThrow(() -> new InvalidTokenException("NToken has an unknown keyId")); - validateSignatureAndExpiration(token, zmsPublicKey); - return token.getPrincipal(); + validateSignatureAndExpiration(principalToken, zmsPublicKey); + return new AthenzPrincipal( + AthenzUtils.createAthenzIdentity( + new AthenzDomain(principalToken.getDomain()), + principalToken.getName()), + token); } private Optional<PublicKey> getPublicKey(String keyId) throws InvalidTokenException { @@ -50,13 +62,13 @@ class NTokenValidator { } } - private static void validateSignatureAndExpiration(NToken token, PublicKey zmsPublicKey) throws InvalidTokenException { - try { - token.validateSignatureAndExpiration(zmsPublicKey); - } catch (InvalidTokenException e) { - // The underlying error message is not user friendly - logDebug(e.getMessage()); - throw new InvalidTokenException("NToken is expired or has invalid signature"); + private static void validateSignatureAndExpiration(PrincipalToken token, + PublicKey zmsPublicKey) throws InvalidTokenException { + StringBuilder errorMessageBuilder = new StringBuilder(); + if (!token.validate(zmsPublicKey, (int) ALLOWED_TIMESTAMP_OFFSET, true, errorMessageBuilder)) { + String message = "NToken is expired or has invalid signature: " + errorMessageBuilder.toString(); + logDebug(message); + throw new InvalidTokenException(message); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java index bfa543f160a..b4859220667 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/UserAuthWithAthenzPrincipalFilter.java @@ -8,13 +8,15 @@ import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; -import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import com.yahoo.vespa.hosted.controller.restapi.application.Authorizer; import java.security.Principal; +import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Logger; import java.util.stream.Stream; @@ -34,11 +36,13 @@ public class UserAuthWithAthenzPrincipalFilter extends AthenzPrincipalFilter { private static final Logger log = Logger.getLogger(UserAuthWithAthenzPrincipalFilter.class.getName()); private final String userAuthenticationPassThruAttribute; + private final String principalHeaderName; @Inject public UserAuthWithAthenzPrincipalFilter(ZmsKeystore zmsKeystore, Executor executor, AthenzConfig config) { super(zmsKeystore, executor, config); this.userAuthenticationPassThruAttribute = config.userAuthenticationPassThruAttribute(); + this.principalHeaderName = config.principalHeaderName(); } @Override @@ -81,13 +85,14 @@ public class UserAuthWithAthenzPrincipalFilter extends AthenzPrincipalFilter { * NOTE: The Bouncer user roles ({@link DiscFilterRequest#roles} are still intact as they are required * for {@link Authorizer#isMemberOfVespaBouncerGroup(HttpRequest)}. */ - private static void rewriteUserPrincipalToAthenz(DiscFilterRequest request) { + private void rewriteUserPrincipalToAthenz(DiscFilterRequest request) { Principal userPrincipal = request.getUserPrincipal(); log.log(LogLevel.DEBUG, () -> "Original user principal: " + userPrincipal.toString()); UserId userId = new UserId(userPrincipal.getName()); - AthenzPrincipal athenzPrincipal = AthenzUtils.createPrincipal(userId); - request.setUserPrincipal(athenzPrincipal); - request.setRemoteUser(athenzPrincipal.toYRN()); + AthenzUser athenzIdentity = AthenzUser.fromUserId(userId); + request.setRemoteUser(athenzIdentity.getFullName()); + NToken nToken = Optional.ofNullable(request.getHeader(principalHeaderName)).map(NToken::new).orElse(null); + request.setUserPrincipal(new AthenzPrincipal(athenzIdentity, nToken)); } private enum UserAuthenticationResult { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 1c32b35f599..a91604f937b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -10,17 +10,17 @@ import com.yahoo.athenz.auth.token.PrincipalToken; import com.yahoo.athenz.auth.util.Crypto; import com.yahoo.athenz.zms.ZMSClient; import com.yahoo.athenz.zts.ZTSClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; import com.yahoo.vespa.hosted.controller.api.integration.security.KeyService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import java.security.PrivateKey; -import java.util.concurrent.TimeUnit; +import java.time.Duration; -import static com.yahoo.vespa.hosted.controller.athenz.AthenzUtils.USER_PRINCIPAL_DOMAIN; +import static com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils.USER_PRINCIPAL_DOMAIN; /** * @author bjorncs @@ -51,7 +51,7 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { */ @Override public ZtsClient createZtsClientWithServicePrincipal() { - return new ZtsClientImpl(new ZTSClient(config.ztsUrl(), createServicePrincipal()), config); + return new ZtsClientImpl(new ZTSClient(config.ztsUrl(), createServicePrincipal()), getServicePrivateKey(), config); } /** @@ -59,7 +59,7 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { */ @Override public ZmsClient createZmsClientWithAuthorizedServiceToken(NToken authorizedServiceToken) { - PrincipalToken signedToken = new PrincipalToken(authorizedServiceToken.getToken()); + PrincipalToken signedToken = new PrincipalToken(authorizedServiceToken.getRawToken()); AthenzConfig.Service service = config.service(); signedToken.signForAuthorizedService( config.domain() + "." + service.name(), service.publicKeyId(), getServicePrivateKey()); @@ -75,8 +75,12 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { // TODO bjorncs: Cache principal token SimpleServiceIdentityProvider identityProvider = new SimpleServiceIdentityProvider( - athenzPrincipalAuthority, config.domain(), service.name(), - getServicePrivateKey(), service.publicKeyId(), /*tokenTimeout*/TimeUnit.HOURS.toSeconds(1)); + athenzPrincipalAuthority, + config.domain(), + service.name(), + getServicePrivateKey(), + service.publicKeyId(), + Duration.ofMinutes(service.credentialsExpiryMinutes()).getSeconds()); return identityProvider.getIdentity(config.domain(), service.name()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzSslContextProviderImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzSslContextProviderImpl.java new file mode 100644 index 00000000000..3a7a72ac8ae --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzSslContextProviderImpl.java @@ -0,0 +1,87 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.impl; + +import com.google.inject.Inject; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentityCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzSslContextProvider; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; +import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.UnrecoverableKeyException; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; + +/** + * @author bjorncs + */ +public class AthenzSslContextProviderImpl implements AthenzSslContextProvider { + + private final AthenzClientFactory clientFactory; + private final AthenzConfig config; + + @Inject + public AthenzSslContextProviderImpl(AthenzClientFactory clientFactory, AthenzConfig config) { + this.clientFactory = clientFactory; + this.config = config; + } + + @Override + public SSLContext get() { + return createSslContext(); + } + + private SSLContext createSslContext() { + try { + SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); + sslContext.init(createKeyManagersWithServiceCertificate(clientFactory.createZtsClientWithServicePrincipal()), + createTrustManagersWithAthenzCa(config), + null); + return sslContext; + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new RuntimeException(e); + } + } + + private static KeyManager[] createKeyManagersWithServiceCertificate(ZtsClient ztsClient) { + try { + AthenzIdentityCertificate identityCertificate = ztsClient.getIdentityCertificate(); + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null); + keyStore.setKeyEntry("athenz-controller-key", + identityCertificate.getPrivateKey(), + new char[0], + new Certificate[]{identityCertificate.getCertificate()}); + KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(keyStore, new char[0]); + return keyManagerFactory.getKeyManagers(); + } catch (KeyStoreException | NoSuchAlgorithmException | UnrecoverableKeyException | CertificateException | IOException e) { + throw new RuntimeException(e); + } + } + + private static TrustManager[] createTrustManagersWithAthenzCa(AthenzConfig config) { + try { + KeyStore trustStore = KeyStore.getInstance("JKS"); + try (FileInputStream in = new FileInputStream(config.athenzCaTrustStore())) { + trustStore.load(in, "changeit".toCharArray()); + } + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustManagerFactory.init(trustStore); + return trustManagerFactory.getTrustManagers(); + } catch (CertificateException | IOException | KeyStoreException | NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java index 110e06b767c..d3fac257583 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java @@ -13,12 +13,12 @@ import com.yahoo.athenz.zms.ZMSClientException; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPublicKey; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPublicKey; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import java.util.Arrays; @@ -49,16 +49,16 @@ public class ZmsClientImpl implements ZmsClient { runOrThrow(() -> { Tenancy tenancy = new Tenancy() .setDomain(tenantDomain.id()) - .setService(service.toFullServiceName()) + .setService(service.getFullName()) .setResourceGroups(Collections.emptyList()); - zmsClient.putTenancy(tenantDomain.id(), service.toFullServiceName(), /*auditref*/null, tenancy); + zmsClient.putTenancy(tenantDomain.id(), service.getFullName(), /*auditref*/null, tenancy); }); } @Override public void deleteTenant(AthenzDomain tenantDomain) { log("deleteTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); - runOrThrow(() -> zmsClient.deleteTenancy(tenantDomain.id(), service.toFullServiceName(), /*auditref*/null)); + runOrThrow(() -> zmsClient.deleteTenancy(tenantDomain.id(), service.getFullName(), /*auditref*/null)); } @Override @@ -66,16 +66,16 @@ public class ZmsClientImpl implements ZmsClient { List<TenantRoleAction> tenantRoleActions = createTenantRoleActions(); log("putProviderResourceGroupRoles(" + "tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s, roleActions=%s)", - tenantDomain, service.getDomain().id(), service.getServiceName(), applicationName, tenantRoleActions); + tenantDomain, service.getDomain().id(), service.getName(), applicationName, tenantRoleActions); runOrThrow(() -> { ProviderResourceGroupRoles resourceGroupRoles = new ProviderResourceGroupRoles() .setDomain(service.getDomain().id()) - .setService(service.getServiceName()) + .setService(service.getName()) .setTenant(tenantDomain.id()) .setResourceGroup(applicationName.id()) .setRoles(tenantRoleActions); zmsClient.putProviderResourceGroupRoles( - tenantDomain.id(), service.getDomain().id(), service.getServiceName(), + tenantDomain.id(), service.getDomain().id(), service.getName(), applicationName.id(), /*auditref*/null, resourceGroupRoles); }); } @@ -83,34 +83,34 @@ public class ZmsClientImpl implements ZmsClient { @Override public void deleteApplication(AthenzDomain tenantDomain, ApplicationId applicationName) { log("deleteProviderResourceGroupRoles(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)", - tenantDomain, service.getDomain().id(), service.getServiceName(), applicationName); + tenantDomain, service.getDomain().id(), service.getName(), applicationName); runOrThrow(() -> { zmsClient.deleteProviderResourceGroupRoles( - tenantDomain.id(), service.getDomain().id(), service.getServiceName(), applicationName.id(), /*auditref*/null); + tenantDomain.id(), service.getDomain().id(), service.getName(), applicationName.id(), /*auditref*/null); }); } @Override public boolean hasApplicationAccess( - AthenzPrincipal principal, ApplicationAction action, AthenzDomain tenantDomain, ApplicationId applicationName) { + AthenzIdentity identity, ApplicationAction action, AthenzDomain tenantDomain, ApplicationId applicationName) { return hasAccess( - action.name(), applicationResourceString(tenantDomain, applicationName), principal); + action.name(), applicationResourceString(tenantDomain, applicationName), identity); } @Override - public boolean hasTenantAdminAccess(AthenzPrincipal principal, AthenzDomain tenantDomain) { - return hasAccess(TenantAction._modify_.name(), tenantResourceString(tenantDomain), principal); + public boolean hasTenantAdminAccess(AthenzIdentity identity, AthenzDomain tenantDomain) { + return hasAccess(TenantAction._modify_.name(), tenantResourceString(tenantDomain), identity); } /** * Used when creating tenancies. As there are no tenancy policies at this point, - * we cannot use {@link #hasTenantAdminAccess(AthenzPrincipal, AthenzDomain)} + * we cannot use {@link #hasTenantAdminAccess(AthenzIdentity, AthenzDomain)} */ @Override - public boolean isDomainAdmin(AthenzPrincipal principal, AthenzDomain domain) { - log("getMembership(domain=%s, role=%s, principal=%s)", domain, "admin", principal); + public boolean isDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { + log("getMembership(domain=%s, role=%s, principal=%s)", domain, "admin", identity); return getOrThrow( - () -> zmsClient.getMembership(domain.id(), "admin", principal.toYRN()).getIsMember()); + () -> zmsClient.getMembership(domain.id(), "admin", identity.getFullName()).getIsMember()); } @Override @@ -127,18 +127,18 @@ public class ZmsClientImpl implements ZmsClient { @Override public AthenzPublicKey getPublicKey(AthenzService service, String keyId) { - log("getPublicKeyEntry(domain=%s, service=%s, keyId=%s)", service.getDomain().id(), service.getServiceName(), keyId); + log("getPublicKeyEntry(domain=%s, service=%s, keyId=%s)", service.getDomain().id(), service.getName(), keyId); return getOrThrow(() -> { - PublicKeyEntry entry = zmsClient.getPublicKeyEntry(service.getDomain().id(), service.getServiceName(), keyId); + PublicKeyEntry entry = zmsClient.getPublicKeyEntry(service.getDomain().id(), service.getName(), keyId); return fromYbase64EncodedKey(entry.getKey(), keyId); }); } @Override public List<AthenzPublicKey> getPublicKeys(AthenzService service) { - log("getServiceIdentity(domain=%s, service=%s)", service.getDomain().id(), service.getServiceName()); + log("getServiceIdentity(domain=%s, service=%s)", service.getDomain().id(), service.getName()); return getOrThrow(() -> { - ServiceIdentity serviceIdentity = zmsClient.getServiceIdentity(service.getDomain().id(), service.getServiceName()); + ServiceIdentity serviceIdentity = zmsClient.getServiceIdentity(service.getDomain().id(), service.getName()); return toAthenzPublicKeys(serviceIdentity.getPublicKeys()); }); } @@ -163,10 +163,11 @@ public class ZmsClientImpl implements ZmsClient { .collect(toList()); } - private boolean hasAccess(String action, String resource, AthenzPrincipal principal) { - log("getAccess(action=%s, resource=%s, principal=%s)", action, resource, principal); + private boolean hasAccess(String action, String resource, AthenzIdentity identity) { + log("getAccess(action=%s, resource=%s, principal=%s)", action, resource, identity); return getOrThrow( - () -> zmsClient.getAccess(action, resource, /*trustDomain*/null, principal.toYRN()).getGranted()); + () -> zmsClient.getAccess(action, resource, /*trustDomain*/null, identity.getFullName()) + .getGranted()); } private static void log(String format, Object... args) { @@ -178,7 +179,7 @@ public class ZmsClientImpl implements ZmsClient { wrappedCode.run(); } catch (ZMSClientException e) { logWarning(e); - throw new ZmsException(e); + throw new ZmsException(e.getCode(), e); } } @@ -187,7 +188,7 @@ public class ZmsClientImpl implements ZmsClient { return wrappedCode.get(); } catch (ZMSClientException e) { logWarning(e); - throw new ZmsException(e); + throw new ZmsException(e.getCode(), e); } } @@ -197,7 +198,7 @@ public class ZmsClientImpl implements ZmsClient { private String resourceStringPrefix(AthenzDomain tenantDomain) { return String.format("%s:service.%s.tenant.%s", - service.getDomain().id(), service.getServiceName(), tenantDomain.id()); + service.getDomain().id(), service.getName(), tenantDomain.id()); } private String tenantResourceString(AthenzDomain tenantDomain) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java index fd58a3daba7..513434f7273 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java @@ -3,11 +3,11 @@ package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.inject.Inject; import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPublicKey; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; -import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPublicKey; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsKeystore; import java.security.PublicKey; import java.util.List; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java index 1111e56c742..a29f2e81fba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java @@ -1,18 +1,27 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.impl; +import com.yahoo.athenz.auth.util.Crypto; +import com.yahoo.athenz.zts.InstanceRefreshRequest; +import com.yahoo.athenz.zts.RoleCertificateRequest; import com.yahoo.athenz.zts.TenantDomains; import com.yahoo.athenz.zts.ZTSClient; import com.yahoo.athenz.zts.ZTSClientException; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZtsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentityCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzRoleCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsException; import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.time.Duration; import java.util.List; +import java.util.function.Supplier; import java.util.logging.Logger; import static java.util.stream.Collectors.toList; @@ -26,25 +35,79 @@ public class ZtsClientImpl implements ZtsClient { private final ZTSClient ztsClient; private final AthenzService service; + private final PrivateKey privateKey; + private final String certificateDnsDomain; + private final Duration certExpiry; - public ZtsClientImpl(ZTSClient ztsClient, AthenzConfig config) { + public ZtsClientImpl(ZTSClient ztsClient, PrivateKey privateKey, AthenzConfig config) { this.ztsClient = ztsClient; this.service = new AthenzService(config.domain(), config.service().name()); + this.privateKey = privateKey; + this.certificateDnsDomain = config.certDnsDomain(); + this.certExpiry = Duration.ofMinutes(config.service().credentialsExpiryMinutes()); } @Override - public List<AthenzDomain> getTenantDomainsForUser(AthenzPrincipal principal) { - log.log(LogLevel.DEBUG, String.format( - "getTenantDomains(domain=%s, username=%s, rolename=admin, service=%s)", - service.getDomain().id(), principal, service.getServiceName())); - try { + public List<AthenzDomain> getTenantDomainsForUser(AthenzIdentity identity) { + return getOrThrow(() -> { + log.log(LogLevel.DEBUG, String.format( + "getTenantDomains(domain=%s, identity=%s, rolename=admin, service=%s)", + service.getDomain().id(), identity.getFullName(), service.getFullName())); TenantDomains domains = ztsClient.getTenantDomains( - service.getDomain().id(), principal.toYRN(), "admin", service.getServiceName()); + service.getDomain().id(), identity.getFullName(), "admin", service.getName()); return domains.getTenantDomainNames().stream() .map(AthenzDomain::new) .collect(toList()); + }); + } + + @Override + public AthenzIdentityCertificate getIdentityCertificate() { + return getOrThrow(() -> { + log.log(LogLevel.DEBUG, + String.format("postInstanceRefreshRequest(service=%s)", service.getFullName())); + InstanceRefreshRequest req = + ZTSClient.generateInstanceRefreshRequest( + service.getDomain().id(), + service.getName(), + privateKey, + certificateDnsDomain, + (int) certExpiry.getSeconds()); + X509Certificate certificate = Crypto.loadX509Certificate( + ztsClient.postInstanceRefreshRequest(service.getDomain().id(), service.getName(), req) + .getCertificate()); + return new AthenzIdentityCertificate(certificate, privateKey); + }); + } + + @Override + public AthenzRoleCertificate getRoleCertificate(AthenzDomain roleDomain, String roleName) { + return getOrThrow(() -> { + log.log(LogLevel.DEBUG, + String.format("postRoleCertificateRequest(service=%s, roleDomain=%s, roleName=%s)", + service.getFullName(), roleDomain.id(), roleName)); + RoleCertificateRequest req = + ZTSClient.generateRoleCertificateRequest( + service.getDomain().id(), + service.getName(), + roleDomain.id(), + roleName, + privateKey, + certificateDnsDomain, + (int)certExpiry.getSeconds()); + X509Certificate roleCertificate = Crypto.loadX509Certificate( + ztsClient.postRoleCertificateRequest(roleDomain.id(), roleName, req) + .getToken()); + return new AthenzRoleCertificate(roleCertificate, privateKey); + }); + } + + private static <T> T getOrThrow(Supplier<T> wrappedCode) { + try { + return wrappedCode.get(); } catch (ZTSClientException e) { - throw new ZtsException(e); + log.warning("Error from Athenz: " + e.getMessage()); + throw new ZtsException(e.getCode(), e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java index d4a2d77c115..52a1f2d477d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.hosted.controller.athenz.mock; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; import java.util.logging.Level; import java.util.logging.Logger; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java index 017e8c7be44..c633d780e30 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzDbMock.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.hosted.controller.athenz.mock; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; import java.util.HashMap; import java.util.HashSet; @@ -26,8 +26,8 @@ public class AthenzDbMock { public static class Domain { public final AthenzDomain name; - public final Set<AthenzPrincipal> admins = new HashSet<>(); - public final Set<AthenzPrincipal> tenantAdmins = new HashSet<>(); + public final Set<AthenzIdentity> admins = new HashSet<>(); + public final Set<AthenzIdentity> tenantAdmins = new HashSet<>(); public final Map<ApplicationId, Application> applications = new HashMap<>(); public boolean isVespaTenant = false; @@ -35,13 +35,13 @@ public class AthenzDbMock { this.name = name; } - public Domain admin(AthenzPrincipal user) { - admins.add(user); + public Domain admin(AthenzIdentity identity) { + admins.add(identity); return this; } - public Domain tenantAdmin(AthenzPrincipal user) { - tenantAdmins.add(user); + public Domain tenantAdmin(AthenzIdentity identity) { + tenantAdmins.add(identity); return this; } @@ -56,7 +56,7 @@ public class AthenzDbMock { public static class Application { - public final Map<ApplicationAction, Set<AthenzPrincipal>> acl = new HashMap<>(); + public final Map<ApplicationAction, Set<AthenzIdentity>> acl = new HashMap<>(); public Application() { acl.put(ApplicationAction.deploy, new HashSet<>()); @@ -64,8 +64,8 @@ public class AthenzDbMock { acl.put(ApplicationAction.write, new HashSet<>()); } - public Application addRoleMember(ApplicationAction action, AthenzPrincipal user) { - acl.get(action).add(user); + public Application addRoleMember(ApplicationAction action, AthenzIdentity identity) { + acl.get(action).add(identity); return this; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java index b2e657eae09..4b50a34094a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -1,15 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.mock; -import com.yahoo.athenz.zms.ZMSClientException; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPublicKey; -import com.yahoo.vespa.hosted.controller.athenz.AthenzService; -import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPublicKey; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import java.util.ArrayList; import java.util.List; @@ -61,28 +60,28 @@ public class ZmsClientMock implements ZmsClient { } @Override - public boolean hasApplicationAccess(AthenzPrincipal principal, ApplicationAction action, AthenzDomain tenantDomain, ApplicationId applicationName) { + public boolean hasApplicationAccess(AthenzIdentity identity, ApplicationAction action, AthenzDomain tenantDomain, ApplicationId applicationName) { log("hasApplicationAccess(principal='%s', action='%s', tenantDomain='%s', applicationName='%s')", - principal, action, tenantDomain, applicationName); + identity, action, tenantDomain, applicationName); AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); AthenzDbMock.Application application = domain.applications.get(applicationName); if (application == null) { throw zmsException(400, "Application '%s' not found", applicationName); } - return domain.admins.contains(principal) || application.acl.get(action).contains(principal); + return domain.admins.contains(identity) || application.acl.get(action).contains(identity); } @Override - public boolean hasTenantAdminAccess(AthenzPrincipal principal, AthenzDomain tenantDomain) { - log("hasTenantAdminAccess(principal='%s', tenantDomain='%s')", principal, tenantDomain); - return isDomainAdmin(principal, tenantDomain) || - getDomainOrThrow(tenantDomain, true).tenantAdmins.contains(principal); + public boolean hasTenantAdminAccess(AthenzIdentity identity, AthenzDomain tenantDomain) { + log("hasTenantAdminAccess(principal='%s', tenantDomain='%s')", identity, tenantDomain); + return isDomainAdmin(identity, tenantDomain) || + getDomainOrThrow(tenantDomain, true).tenantAdmins.contains(identity); } @Override - public boolean isDomainAdmin(AthenzPrincipal principal, AthenzDomain domain) { - log("isDomainAdmin(principal='%s', domain='%s')", principal, domain); - return getDomainOrThrow(domain, false).admins.contains(principal); + public boolean isDomainAdmin(AthenzIdentity identity, AthenzDomain domain) { + log("isDomainAdmin(principal='%s', domain='%s')", identity, domain); + return getDomainOrThrow(domain, false).admins.contains(identity); } @Override @@ -111,7 +110,7 @@ public class ZmsClientMock implements ZmsClient { } private static ZmsException zmsException(int code, String message, Object... args) { - return new ZmsException(new ZMSClientException(code, String.format(message, args))); + return new ZmsException(code, String.format(message, args)); } private static void log(String format, Object... args) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java index f21bc011273..d778fb550ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java @@ -1,10 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.athenz.mock; +import com.yahoo.athenz.auth.util.Crypto; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentityCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzRoleCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZtsClient; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.pkcs.PKCS10CertificationRequest; +import java.io.IOException; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.security.cert.X509Certificate; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -24,11 +35,51 @@ public class ZtsClientMock implements ZtsClient { } @Override - public List<AthenzDomain> getTenantDomainsForUser(AthenzPrincipal principal) { - log.log(Level.INFO, "getTenantDomainsForUser(principal='%s')", principal); + public List<AthenzDomain> getTenantDomainsForUser(AthenzIdentity identity) { + log.log(Level.INFO, "getTenantDomainsForUser(principal='%s')", identity); return athenz.domains.values().stream() - .filter(domain -> domain.tenantAdmins.contains(principal) || domain.admins.contains(principal)) + .filter(domain -> domain.tenantAdmins.contains(identity) || domain.admins.contains(identity)) .map(domain -> domain.name) .collect(toList()); } + + @Override + public AthenzIdentityCertificate getIdentityCertificate() { + log.log(Level.INFO, "getIdentityCertificate()"); + try { + KeyPair keyPair = createKeyPair(); + String subject = "CN=controller"; + return new AthenzIdentityCertificate(createCertificate(keyPair, subject), keyPair.getPrivate()); + } catch (NoSuchAlgorithmException | OperatorCreationException | IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public AthenzRoleCertificate getRoleCertificate(AthenzDomain roleDomain, String roleName) { + log.log(Level.INFO, + String.format("getRoleCertificate(roleDomain=%s, roleName=%s)", roleDomain.id(), roleDomain)); + try { + KeyPair keyPair = createKeyPair(); + String subject = String.format("CN=%s:role.%s", roleDomain.id(), roleName); + return new AthenzRoleCertificate(createCertificate(keyPair, subject), keyPair.getPrivate()); + } catch (NoSuchAlgorithmException | OperatorCreationException | IOException e) { + throw new RuntimeException(e); + } + } + + private static X509Certificate createCertificate(KeyPair keyPair, String subject) throws + OperatorCreationException, IOException { + PKCS10CertificationRequest csr = + Crypto.getPKCS10CertRequest( + Crypto.generateX509CSR(keyPair.getPrivate(), subject, null)); + return Crypto.generateX509Certificate(csr, keyPair.getPrivate(), new X500Name(subject), 3600, false); + } + + private static KeyPair createKeyPair() throws NoSuchAlgorithmException { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(512); + return keyGen.genKeyPair(); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java index 7c06ef27ce9..2bf64571bdf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedApplication; @@ -46,7 +46,6 @@ public class DeploymentOrder { /** Returns a list of jobs to trigger after the given job */ // TODO: This does too much - should just tell us the order, as advertised - // TODO: You're next! public List<JobType> nextAfter(JobType job, LockedApplication application) { if ( ! application.deploying().isPresent()) { // Change was cancelled return Collections.emptyList(); @@ -106,9 +105,9 @@ public class DeploymentOrder { /** Returns deployments sorted according to declared zones */ public List<Deployment> sortBy(List<DeploymentSpec.DeclaredZone> zones, Collection<Deployment> deployments) { - List<Zone> productionZones = zones.stream() + List<ZoneId> productionZones = zones.stream() .filter(z -> z.region().isPresent()) - .map(z -> new Zone(z.environment(), z.region().get())) + .map(z -> ZoneId.from(z.environment(), z.region().get())) .collect(toList()); return deployments.stream() .sorted(comparingInt(deployment -> productionZones.indexOf(deployment.zone()))) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 1eee727214b..f0c950b024b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -4,8 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.curator.Lock; +import com.yahoo.config.provision.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -28,7 +27,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Consumer; import java.util.logging.Logger; /** @@ -79,7 +77,7 @@ public class DeploymentTrigger { * @param report information about the job that just completed */ public void triggerFromCompletion(JobReport report) { - applications().lockedOrThrow(report.applicationId(), application -> { + applications().lockOrThrow(report.applicationId(), application -> { application = application.withJobCompletion(report, clock.instant(), controller); // Handle successful starting and ending @@ -132,7 +130,7 @@ public class DeploymentTrigger { ApplicationList applications = ApplicationList.from(applications().asList()); applications = applications.notPullRequest(); for (Application application : applications.asList()) - applications().lockedIfPresent(application.id(), this::triggerReadyJobs); + applications().lockIfPresent(application.id(), this::triggerReadyJobs); } /** Find the next step to trigger if any, and triggers it */ @@ -219,7 +217,7 @@ public class DeploymentTrigger { * @throws IllegalArgumentException if this application already have an ongoing change */ public void triggerChange(ApplicationId applicationId, Change change) { - applications().lockedOrThrow(applicationId, application -> { + applications().lockOrThrow(applicationId, application -> { if (application.deploying().isPresent() && ! application.deploymentJobs().hasFailures()) throw new IllegalArgumentException("Could not start " + change + " on " + application + ": " + application.deploying().get() + " is already in progress"); @@ -238,7 +236,7 @@ public class DeploymentTrigger { * @param applicationId the application to trigger */ public void cancelChange(ApplicationId applicationId) { - applications().lockedOrThrow(applicationId, application -> { + applications().lockOrThrow(applicationId, application -> { buildSystem.removeJobs(application.id()); applications().store(application.withDeploying(Optional.empty())); }); @@ -360,7 +358,7 @@ public class DeploymentTrigger { */ private boolean isOnNewerVersionInProductionThan(Version version, Application application, JobType job) { if ( ! job.isProduction()) return false; - Optional<Zone> zone = job.zone(controller.system()); + Optional<ZoneId> zone = job.zone(controller.system()); if ( ! zone.isPresent()) return false; Deployment existingDeployment = application.deployments().get(zone.get()); if (existingDeployment == null) return false; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 09f8df58205..2b7260d5ffa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -1,7 +1,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.Tenant; @@ -86,7 +85,7 @@ public class ApplicationOwnershipConfirmer extends Maintainer { } protected void store(IssueId issueId, ApplicationId applicationId) { - controller().applications().lockedIfPresent(applicationId, application -> + controller().applications().lockIfPresent(applicationId, application -> controller().applications().store(application.withOwnershipIssueId(issueId))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index ae617f87be6..ad7fa90967b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -4,10 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.curator.Lock; +import com.yahoo.config.provision.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList; import com.yahoo.vespa.hosted.controller.application.ApplicationList; @@ -46,7 +45,7 @@ public class ClusterInfoMaintainer extends Maintainer { return node.membership.clusterId; } - private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeList nodes, Zone zone) { + private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeList nodes, ZoneId zone) { Map<ClusterSpec.Id, ClusterInfo> infoMap = new HashMap<>(); // Group nodes by clusterid @@ -65,7 +64,8 @@ public class ClusterInfoMaintainer extends Maintainer { double cpu = 0; double mem = 0; double disk = 0; - if (zone.nodeFlavors().isPresent()) { + // TODO: This code was never run. Reenable when flavours are available from a FlavorRegistry or something, or remove. + /*if (zone.nodeFlavors().isPresent()) { Optional<Flavor> flavorOptional = zone.nodeFlavors().get().getFlavor(node.flavor); if ((flavorOptional.isPresent())) { Flavor flavor = flavorOptional.get(); @@ -73,7 +73,7 @@ public class ClusterInfoMaintainer extends Maintainer { mem = flavor.getMinMainMemoryAvailableGb(); disk = flavor.getMinMainMemoryAvailableGb(); } - } + }*/ // Add to map List<String> hostnames = clusterNodes.stream().map(node1 -> node1.hostname).collect(Collectors.toList()); @@ -93,7 +93,7 @@ public class ClusterInfoMaintainer extends Maintainer { try { NodeList nodes = controller().applications().configserverClient().getNodeList(deploymentId); Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes, deployment.zone()); - controller().applications().lockedIfPresent(application.id(), lockedApplication -> + controller().applications().lockIfPresent(application.id(), lockedApplication -> controller.applications().store(lockedApplication.withClusterInfo(deployment.zone(), clusterInfo))); } catch (IOException | IllegalArgumentException e) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index 3744be67135..58e32344372 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -3,8 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.curator.Lock; +import com.yahoo.config.provision.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; @@ -30,7 +29,7 @@ public class ClusterUtilizationMaintainer extends Maintainer { this.controller = controller; } - private Map<ClusterSpec.Id, ClusterUtilization> getUpdatedClusterUtilizations(ApplicationId app, Zone zone) { + private Map<ClusterSpec.Id, ClusterUtilization> getUpdatedClusterUtilizations(ApplicationId app, ZoneId zone) { Map<String, MetricsService.SystemMetrics> systemMetrics = controller.metricsService().getSystemMetrics(app, zone); Map<ClusterSpec.Id, ClusterUtilization> utilizationMap = new HashMap<>(); @@ -50,7 +49,7 @@ public class ClusterUtilizationMaintainer extends Maintainer { Map<ClusterSpec.Id, ClusterUtilization> clusterUtilization = getUpdatedClusterUtilizations(application.id(), deployment.zone()); - controller().applications().lockedIfPresent(application.id(), lockedApplication -> + controller().applications().lockIfPresent(application.id(), lockedApplication -> controller().applications().store(lockedApplication.withClusterUtilization(deployment.zone(), clusterUtilization))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index bc2112ac0ca..4e9dd94d8e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; @@ -34,11 +35,13 @@ public class ControllerMaintenance extends AbstractComponent { private final ClusterUtilizationMaintainer clusterUtilizationMaintainer; private final DeploymentMetricsMaintainer deploymentMetricsMaintainer; private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer; + private final DnsMaintainer dnsMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator, JobControl jobControl, Metric metric, Chef chefClient, - DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues) { + DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, + NameService nameService) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); this.jobControl = jobControl; deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl); @@ -52,6 +55,7 @@ public class ControllerMaintenance extends AbstractComponent { clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); + dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(1), jobControl, nameService); } public Upgrader upgrader() { return upgrader; } @@ -72,6 +76,7 @@ public class ControllerMaintenance extends AbstractComponent { clusterInfoMaintainer.deconstruct(); deploymentMetricsMaintainer.deconstruct(); applicationOwnershipConfirmer.deconstruct(); + dnsMaintainer.deconstruct(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index ae6ba364d25..324868878af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.Tenant; @@ -131,7 +130,7 @@ public class DeploymentIssueReporter extends Maintainer { } private void store(ApplicationId id, IssueId issueId) { - controller().applications().lockedIfPresent(id, application -> + controller().applications().lockIfPresent(id, application -> controller().applications().store(application.withDeploymentIssueId(issueId))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 13eb5075f34..821efba013d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; @@ -34,7 +33,7 @@ public class DeploymentMetricsMaintainer extends Maintainer { boolean hasWarned = false; for (Application application : ApplicationList.from(controller().applications().asList()).notPullRequest().asList()) { try { - controller().applications().lockedIfPresent(application.id(), lockedApplication -> + controller().applications().lockIfPresent(application.id(), lockedApplication -> controller().applications().store(lockedApplication.with(controller().metricsService().getApplicationMetrics(application.id())))); for (Deployment deployment : application.deployments().values()) { @@ -46,7 +45,7 @@ public class DeploymentMetricsMaintainer extends Maintainer { deploymentMetrics.queryLatencyMillis(), deploymentMetrics.writeLatencyMillis()); - controller().applications().lockedIfPresent(application.id(), lockedApplication -> + controller().applications().lockIfPresent(application.id(), lockedApplication -> controller().applications().store(lockedApplication.with(deployment.zone(), appMetrics))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java new file mode 100644 index 00000000000..89394bf4dd9 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java @@ -0,0 +1,67 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.application.ApplicationRotation; +import com.yahoo.vespa.hosted.controller.rotation.Rotation; +import com.yahoo.vespa.hosted.controller.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; + +import java.time.Duration; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; + +/** + * Performs DNS maintenance tasks such as removing DNS aliases for unassigned rotations. + * + * @author mpolden + */ +public class DnsMaintainer extends Maintainer { + + private static final Logger log = Logger.getLogger(DnsMaintainer.class.getName()); + + private final NameService nameService; + + public DnsMaintainer(Controller controller, Duration interval, JobControl jobControl, + NameService nameService) { + super(controller, interval, jobControl); + this.nameService = nameService; + } + + private RotationRepository rotationRepository() { + return controller().applications().rotationRepository(); + } + + @Override + protected void maintain() { + try (RotationLock lock = rotationRepository().lock()) { + Map<RotationId, Rotation> unassignedRotations = rotationRepository().availableRotations(lock); + unassignedRotations.values().forEach(this::removeDnsAlias); + } + } + + /** Remove DNS alias for unassigned rotation */ + private void removeDnsAlias(Rotation rotation) { + // When looking up CNAME by data, the data must be a FQDN + Optional<Record> record = nameService.findRecord(Record.Type.CNAME, RecordData.fqdn(rotation.name())); + record.filter(this::canUpdate) + .ifPresent(r -> { + log.warning(String.format("Want to remove DNS record %s (%s) because it points to the unassigned " + + "rotation %s (%s)", record.get().id().asString(), + record.get().name().asString(), rotation.id().asString(), rotation.name())); + // TODO: Actually remove the record + //nameService.removeRecord(r.id()); + }); + } + + /** Returns whether we can update the given record */ + private boolean canUpdate(Record record) { + return record.name().asString().endsWith(ApplicationRotation.DNS_SUFFIX); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 01e53ce4f79..ab388ca9a9f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; import com.yahoo.vespa.hosted.controller.application.ApplicationList; +import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import java.time.Clock; import java.time.Duration; @@ -23,11 +24,14 @@ import java.util.Optional; /** * @author mortent + * @author mpolden */ public class MetricsReporter extends Maintainer { public static final String convergeMetric = "seconds.since.last.chef.convergence"; public static final String deploymentFailMetric = "deployment.failurePercentage"; + public static final String remainingRotations = "remaining_rotations"; + private final Metric metric; private final Chef chefClient; private final Clock clock; @@ -51,6 +55,14 @@ public class MetricsReporter extends Maintainer { public void maintain() { reportChefMetrics(); reportDeploymentMetrics(); + reportRemainingRotations(); + } + + private void reportRemainingRotations() { + try (RotationLock lock = controller().applications().rotationRepository().lock()) { + int availableRotations = controller().applications().rotationRepository().availableRotations(lock).size(); + metric.set(remainingRotations, availableRotations, metric.createContext(Collections.emptyMap())); + } } private void reportChefMetrics() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 23316a74aae..762f12c3e8a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.ArrayList; @@ -55,6 +56,7 @@ public class ApplicationSerializer { private final String ownershipIssueIdField = "ownershipIssueId"; private final String writeQualityField = "writeQuality"; private final String queryQualityField = "queryQuality"; + private final String rotationField = "rotation"; // Deployment fields private final String zoneField = "zone"; @@ -114,7 +116,7 @@ public class ApplicationSerializer { private final String deploymentMetricsQueryLatencyField = "queryLatencyMillis"; private final String deploymentMetricsWriteLatencyField = "writeLatencyMillis"; - + // ------------------ Serialization public Slime toSlime(Application application) { @@ -130,6 +132,7 @@ public class ApplicationSerializer { application.ownershipIssueId().ifPresent(issueId -> root.setString(ownershipIssueIdField, issueId.value())); root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); + application.rotation().ifPresent(rotation -> root.setString(rotationField, rotation.id().asString())); return slime; } @@ -139,7 +142,7 @@ public class ApplicationSerializer { } private void deploymentToSlime(Deployment deployment, Cursor object) { - zoneToSlime(deployment.zone(), object.setObject(zoneField)); + zoneIdToSlime(deployment.zone(), object.setObject(zoneField)); object.setString(versionField, deployment.version().toString()); object.setLong(deployTimeField, deployment.at().toEpochMilli()); toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); @@ -191,7 +194,7 @@ public class ApplicationSerializer { object.setDouble(clusterUtilsDiskBusyField, utils.getDiskBusy()); } - private void zoneToSlime(Zone zone, Cursor object) { + private void zoneIdToSlime(ZoneId zone, Cursor object) { object.setString(environmentField, zone.environment().value()); object.setString(regionField, zone.region().value()); } @@ -267,9 +270,10 @@ public class ApplicationSerializer { Optional<IssueId> ownershipIssueId = optionalString(root.field(ownershipIssueIdField)).map(IssueId::from); ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble()); + Optional<RotationId> rotation = rotationFromSlime(root.field(rotationField)); - return new Application(id, deploymentSpec, validationOverrides, deployments, - deploymentJobs, deploying, outstandingChange, ownershipIssueId, metrics); + return new Application(id, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, + outstandingChange, ownershipIssueId, metrics, rotation); } private List<Deployment> deploymentsFromSlime(Inspector array) { @@ -279,13 +283,13 @@ public class ApplicationSerializer { } private Deployment deploymentFromSlime(Inspector deploymentObject) { - return new Deployment(zoneFromSlime(deploymentObject.field(zoneField)), + return new Deployment(zoneIdFromSlime(deploymentObject.field(zoneField)), applicationRevisionFromSlime(deploymentObject.field(applicationPackageRevisionField)).get(), Version.fromString(deploymentObject.field(versionField).asString()), Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()), - clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)), - clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)), - deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField))); + clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)), + clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)), + deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField))); } private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) { @@ -334,9 +338,8 @@ public class ApplicationSerializer { return new ClusterInfo(flavor, cost, flavorCpu, flavorMem, flavorDisk, ClusterSpec.Type.from(type), hostnames); } - private Zone zoneFromSlime(Inspector object) { - return new Zone(Environment.from(object.field(environmentField).asString()), - RegionName.from(object.field(regionField).asString())); + private ZoneId zoneIdFromSlime(Inspector object) { + return ZoneId.from(object.field(environmentField).asString(), object.field(regionField).asString()); } private Optional<ApplicationRevision> applicationRevisionFromSlime(Inspector object) { @@ -403,6 +406,10 @@ public class ApplicationSerializer { Instant.ofEpochMilli(object.field(atField).asLong()))); } + private Optional<RotationId> rotationFromSlime(Inspector field) { + return field.valid() ? optionalString(field).map(RotationId::new) : Optional.empty(); + } + private Optional<Long> optionalLong(Inspector field) { return field.valid() ? Optional.of(field.asLong()) : Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java index 3fbfdd31808..fb6608ea643 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ControllerDb.java @@ -6,12 +6,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; -import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import java.util.List; import java.util.Optional; -import java.util.Set; /** * Used to store the permanent data of the controller. @@ -19,55 +17,46 @@ import java.util.Set; * @author Stian Kristoffersen * @author bratseth */ -public abstract class ControllerDb { +public interface ControllerDb { // --------- Tenants - public abstract void createTenant(Tenant tenant); + void createTenant(Tenant tenant); - public abstract void updateTenant(Tenant tenant) throws PersistenceException; + // TODO: Remove exception from all signatures + void updateTenant(Tenant tenant) throws PersistenceException; - public abstract void deleteTenant(TenantId tenantId) throws PersistenceException; + void deleteTenant(TenantId tenantId) throws PersistenceException; - public abstract Optional<Tenant> getTenant(TenantId tenantId) throws PersistenceException; + Optional<Tenant> getTenant(TenantId tenantId) throws PersistenceException; - public abstract List<Tenant> listTenants(); + List<Tenant> listTenants(); // --------- Applications // ONLY call this from ApplicationController.store() - public abstract void store(Application application); + void store(Application application); - public abstract void deleteApplication(ApplicationId applicationId); + void deleteApplication(ApplicationId applicationId); - public abstract Optional<Application> getApplication(ApplicationId applicationId); + Optional<Application> getApplication(ApplicationId applicationId); /** Returns all applications */ - public abstract List<Application> listApplications(); + List<Application> listApplications(); /** Returns all applications of a tenant */ - public abstract List<Application> listApplications(TenantId tenantId); - - // --------- Rotations - - public abstract Set<RotationId> getRotations(); - - public abstract Set<RotationId> getRotations(ApplicationId applicationId); - - public abstract boolean assignRotation(RotationId rotationId, ApplicationId applicationId); - - public abstract Set<RotationId> deleteRotations(ApplicationId applicationId); + List<Application> listApplications(TenantId tenantId); /** Returns the given elements joined by dot "." */ - protected String path(Identifier... elements) { + default String path(Identifier... elements) { return Joiner.on(".").join(elements); } - protected String path(String... elements) { + default String path(String... elements) { return Joiner.on(".").join(elements); } - protected String path(ApplicationId applicationId) { + default String path(ApplicationId applicationId) { return applicationId.tenant().value() + "." + applicationId.application().value() + "." + applicationId.instance().value(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index e5616f025ce..a3bb191fc38 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -39,6 +39,8 @@ public class CuratorDb { private static final Path root = Path.fromString("/controller/v1"); + private static final Path lockRoot = root.append("locks"); + private static final Duration defaultLockTimeout = Duration.ofMinutes(5); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); @@ -67,6 +69,10 @@ public class CuratorDb { return lock(lockPath(id), timeout); } + public Lock lockRotations() { + return lock(lockRoot.append("rotations"), defaultLockTimeout); + } + /** Create a reentrant lock */ private Lock lock(Path path, Duration timeout) { Lock lock = locks.computeIfAbsent(path, (pathArg) -> new Lock(pathArg.getAbsolute(), curator)); @@ -75,18 +81,18 @@ public class CuratorDb { } public Lock lockInactiveJobs() { - return lock(root.append("locks").append("inactiveJobsLock"), defaultLockTimeout); + return lock(lockRoot.append("inactiveJobsLock"), defaultLockTimeout); } public Lock lockJobQueues() { - return lock(root.append("locks").append("jobQueuesLock"), defaultLockTimeout); + return lock(lockRoot.append("jobQueuesLock"), defaultLockTimeout); } public Lock lockMaintenanceJob(String jobName) { // Use a short timeout such that if maintenance jobs are started at about the same time on different nodes // and the maintenance job takes a long time to complete, only one of the nodes will run the job // in each maintenance interval - return lock(root.append("locks").append("maintenanceJobLocks").append(jobName), Duration.ofSeconds(1)); + return lock(lockRoot.append("maintenanceJobLocks").append(jobName), Duration.ofSeconds(1)); } public Lock lockProvisionState(String provisionStateId) { @@ -94,11 +100,11 @@ public class CuratorDb { } public Lock lockVespaServerPool() { - return lock(root.append("locks").append("vespaServerPoolLock"), Duration.ofSeconds(1)); + return lock(lockRoot.append("vespaServerPoolLock"), Duration.ofSeconds(1)); } public Lock lockOpenStackServerPool() { - return lock(root.append("locks").append("openStackServerPoolLock"), Duration.ofSeconds(1)); + return lock(lockRoot.append("openStackServerPoolLock"), Duration.ofSeconds(1)); } // -------------- Read and write -------------------------------------------------- @@ -222,19 +228,15 @@ public class CuratorDb { // -------------- Paths -------------------------------------------------- - private Path systemVersionPath() { - return root.append("systemVersion"); - } - private Path lockPath(TenantId tenant) { - Path lockPath = root.append("locks") + Path lockPath = lockRoot .append(tenant.id()); curator.create(lockPath); return lockPath; } private Path lockPath(ApplicationId application) { - Path lockPath = root.append("locks") + Path lockPath = lockRoot .append(application.tenant().value()) .append(application.application().value()) .append(application.instance().value()); @@ -243,7 +245,7 @@ public class CuratorDb { } private Path lockPath(String provisionId) { - Path lockPath = root.append("locks") + Path lockPath = lockRoot .append(provisionStatePath()) .append(provisionId); curator.create(lockPath); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java index ab240b9dea9..2c5d77c7773 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MemoryControllerDb.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.hosted.controller.AlreadyExistsException; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.Tenant; -import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import java.util.ArrayList; @@ -14,7 +13,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; /** @@ -22,11 +20,10 @@ import java.util.stream.Collectors; * * @author Stian Kristoffersen */ -public class MemoryControllerDb extends ControllerDb { +public class MemoryControllerDb implements ControllerDb { private final Map<TenantId, Tenant> tenants = new HashMap<>(); private final Map<String, Application> applications = new HashMap<>(); - private final Map<RotationId, ApplicationId> rotationAssignments = new HashMap<>(); @Override public void createTenant(Tenant tenant) { @@ -52,7 +49,7 @@ public class MemoryControllerDb extends ControllerDb { } @Override - public Optional<Tenant> getTenant(TenantId tenantId) throws PersistenceException { + public Optional<Tenant> getTenant(TenantId tenantId) { return Optional.ofNullable(tenants.get(tenantId)); } @@ -88,36 +85,4 @@ public class MemoryControllerDb extends ControllerDb { .collect(Collectors.toList()); } - @Override - public Set<RotationId> getRotations() { - return rotationAssignments.keySet(); - } - - @Override - public Set<RotationId> getRotations(ApplicationId applicationId) { - return rotationAssignments.entrySet().stream() - .filter(entry -> entry.getValue().equals(applicationId)) - .map(Map.Entry::getKey) - .collect(Collectors.toSet()); - } - - @Override - public boolean assignRotation(RotationId rotationId, ApplicationId applicationId) { - if (rotationAssignments.containsKey(rotationId)) { - return false; - } else { - rotationAssignments.put(rotationId, applicationId); - return true; - } - } - - @Override - public Set<RotationId> deleteRotations(ApplicationId applicationId) { - Set<RotationId> rotations = getRotations(applicationId); - for (RotationId rotation : rotations) { - rotationAssignments.remove(rotation); - } - return rotations; - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index e8b68d0c55a..c9ce0b76520 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; @@ -83,8 +83,8 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { ObjectMapper mapper = new ObjectMapper(); DiscoveryResponseStructure responseStructure = new DiscoveryResponseStructure(); - List<Zone> zones = zoneRegistry.zones(); - for (Zone zone : zones) { + List<ZoneId> zones = zoneRegistry.zones(); + for (ZoneId zone : zones) { if (!"".equals(proxyRequest.getEnvironment()) && !proxyRequest.getEnvironment().equals(zone.environment().value())) { continue; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d7324450d4c..4e3b96b40df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -21,7 +21,6 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.AlreadyExistsException; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -53,7 +52,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; -import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -66,9 +64,12 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.NToken; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.Path; @@ -93,7 +94,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Scanner; -import java.util.Set; import java.util.concurrent.Executor; import java.util.logging.Level; @@ -241,7 +241,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { String userIdString = request.getProperty("userOverride"); if (userIdString == null) userIdString = userFrom(request) - .orElseThrow(() -> new ForbiddenException("You must be authenticated or specify userOverride")); + .map(UserId::id) + .orElseThrow(() -> new ForbiddenException("You must be authenticated or specify userOverride")); UserId userId = new UserId(userIdString); List<Tenant> tenants = controller.tenants().asList(userId); @@ -376,13 +377,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Compile version. The version that should be used when building an application object.setString("compileVersion", application.oldestDeployedVersion().orElse(controller.systemVersion()).toFullString()); - // Rotations + // Rotation Cursor globalRotationsArray = object.setArray("globalRotations"); - Set<URI> rotations = controller.getRotationUris(application.id()); - Map<String, RotationStatus> rotationHealthStatus = - rotations.isEmpty() ? Collections.emptyMap() : controller.getHealthStatus(rotations.iterator().next().getHost()); - for (URI rotation : rotations) - globalRotationsArray.addString(rotation.toString()); + Map<String, RotationStatus> rotationHealthStatus = application.rotation() + .map(rotation -> controller.getHealthStatus(rotation.dnsName())) + .orElse(Collections.emptyMap()); + application.rotation().ifPresent(rotation -> globalRotationsArray.addString(rotation.url().toString())); // Deployments sorted according to deployment spec List<Deployment> deployments = controller.applications().deploymentTrigger() @@ -395,7 +395,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { deploymentObject.setString("environment", deployment.zone().environment().value()); deploymentObject.setString("region", deployment.zone().region().value()); deploymentObject.setString("instance", application.id().instance().value()); // pointless - if ( ! rotations.isEmpty()) + if (application.rotation().isPresent()) setRotationStatus(deployment, rotationHealthStatus, deploymentObject); if (recurseOverDeployments(request)) // List full deployment information when recursive. @@ -423,7 +423,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .orElseThrow(() -> new NotExistsException(id + " not found")); DeploymentId deploymentId = new DeploymentId(application.id(), - new Zone(Environment.from(environment), RegionName.from(region))); + ZoneId.from(environment, region)); Deployment deployment = application.deployments().get(deploymentId.zone()); if (deployment == null) @@ -507,14 +507,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector requestData = toSlime(request.getData()).get(); String reason = mandatory("reason", requestData).asString(); - String agent = authorizer.getUserId(request).toString(); + String agent = authorizer.getIdentity(request).getFullName(); long timestamp = controller.clock().instant().getEpochSecond(); EndpointStatus.Status status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; EndpointStatus endPointStatus = new EndpointStatus(status, reason, agent, timestamp); // DeploymentId identifies the zone and application we are dealing with DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - new Zone(Environment.from(environment), RegionName.from(region))); + ZoneId.from(environment, region)); try { List<String> rotations = controller.applications().setGlobalRotationStatus(deploymentId, endPointStatus); return new MessageResponse(String.format("Rotations %s successfully set to %s service", rotations.toString(), inService ? "in" : "out of")); @@ -526,7 +526,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - new Zone(Environment.from(environment), RegionName.from(region))); + ZoneId.from(environment, region)); Slime slime = new Slime(); Cursor c1 = slime.setObject().setArray("globalrotationoverride"); @@ -549,17 +549,16 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse rotationStatus(String tenantName, String applicationName, String instanceName, String environment, String region) { - ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); - Set<URI> rotations = controller.getRotationUris(applicationId); - if (rotations.isEmpty()) + Application application = controller.applications().require(applicationId); + if (!application.rotation().isPresent()) { throw new NotExistsException("global rotation does not exist for '" + environment + "." + region + "'"); + } Slime slime = new Slime(); Cursor response = slime.setObject(); - Map<String, RotationStatus> rotationHealthStatus = controller.getHealthStatus(rotations.iterator().next().getHost()); - + Map<String, RotationStatus> rotationHealthStatus = controller.getHealthStatus(application.rotation().get().dnsName()); for (String rotationEndpoint : rotationHealthStatus.keySet()) { if (rotationEndpoint.contains(toDns(environment)) && rotationEndpoint.contains(toDns(region))) { Cursor bcpStatusObject = response.setObject("bcpStatus"); @@ -572,13 +571,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse waitForConvergence(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { return new JacksonJsonResponse(controller.waitForConfigConvergence(new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - new Zone(Environment.from(environment), RegionName.from(region))), + ZoneId.from(environment, region)), asLong(request.getProperty("timeout"), 1000))); } private HttpResponse services(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationView applicationView = controller.getApplicationView(tenantName, applicationName, instanceName, environment, region); - ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.from(environment), RegionName.from(region)), + ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), controller.getConfigServerUris(Environment.from(environment), RegionName.from(region)), request.getUri()); @@ -588,7 +587,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse service(String tenantName, String applicationName, String instanceName, String environment, String region, String serviceName, String restPath, HttpRequest request) { Map<?,?> result = controller.getServiceApiResponse(tenantName, applicationName, instanceName, environment, region, serviceName, restPath); - ServiceApiResponse response = new ServiceApiResponse(new Zone(Environment.from(environment), RegionName.from(region)), + ServiceApiResponse response = new ServiceApiResponse(ZoneId.from(environment, region), new ApplicationId.Builder().tenant(tenantName).applicationName(applicationName).instanceName(instanceName).build(), controller.getConfigServerUris(Environment.from(environment), RegionName.from(region)), request.getUri()); @@ -597,15 +596,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse createUser(HttpRequest request) { - Optional<String> username = userFrom(request); - if ( ! username.isPresent() ) throw new ForbiddenException("Not authenticated."); + Optional<UserId> user = userFrom(request); + if ( ! user.isPresent() ) throw new ForbiddenException("Not authenticated."); try { - controller.tenants().createUserTenant(username.get()); - return new MessageResponse("Created user '" + username.get() + "'"); + controller.tenants().createUserTenant(user.get().id()); + return new MessageResponse("Created user '" + user.get() + "'"); } catch (AlreadyExistsException e) { // Ok - return new MessageResponse("User '" + username + "' already exists"); + return new MessageResponse("User '" + user + "' already exists"); } } @@ -711,7 +710,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { "Active versions: " + controller.versionStatus().versions()); ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); - controller.applications().lockedOrThrow(id, application -> { + controller.applications().lockOrThrow(id, application -> { if (application.deploying().isPresent()) throw new IllegalArgumentException("Can not start a deployment of " + application + " at this time: " + application.deploying().get() + " is in progress"); @@ -729,7 +728,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if ( ! change.isPresent()) return new MessageResponse("No deployment in progress for " + application + " at this time"); - controller.applications().lockedOrThrow(id, lockedApplication -> + controller.applications().lockOrThrow(id, lockedApplication -> controller.applications().deploymentTrigger().cancelChange(id)); return new MessageResponse("Cancelled " + change.get() + " for " + application); @@ -738,12 +737,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { /** Schedule restart of deployment, or specific host in a deployment */ private HttpResponse restart(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - new Zone(Environment.from(environment), RegionName.from(region))); + ZoneId.from(environment, region)); // TODO: Propagate all filters - if (request.getProperty("hostname") != null) - controller.applications().restartHost(deploymentId, new Hostname(request.getProperty("hostname"))); - else - controller.applications().restart(deploymentId); + Optional<Hostname> hostname = Optional.ofNullable(request.getProperty("hostname")).map(Hostname::new); + controller.applications().restart(deploymentId, hostname); // TODO: Change to return JSON return new StringResponse("Requested restart of " + path(TenantResource.API_PATH, tenantName, @@ -761,7 +758,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse log(String tenantName, String applicationName, String instanceName, String environment, String region) { try { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), - new Zone(Environment.from(environment), RegionName.from(region))); + ZoneId.from(environment, region)); return new JacksonJsonResponse(controller.grabLog(deploymentId)); } catch (RuntimeException e) { @@ -773,7 +770,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deploy(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); - Zone zone = new Zone(Environment.from(environment), RegionName.from(region)); + ZoneId zone = ZoneId.from(environment, region); Map<String, byte[]> dataParts = new MultipartParser().parse(request); if ( ! dataParts.containsKey("deployOptions")) @@ -783,17 +780,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); + ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get("applicationZip")); DeployAuthorizer deployAuthorizer = new DeployAuthorizer(controller.zoneRegistry(), athenzClientFactory); Tenant tenant = controller.tenants().tenant(new TenantId(tenantName)).orElseThrow(() -> new NotExistsException(new TenantId(tenantName))); Principal principal = authorizer.getPrincipal(request); - deployAuthorizer.throwIfUnauthorizedForDeploy(principal, Environment.from(environment), tenant, applicationId); + deployAuthorizer.throwIfUnauthorizedForDeploy(principal, Environment.from(environment), tenant, applicationId, applicationPackage); // TODO: get rid of the json object DeployOptions deployOptionsJsonClass = new DeployOptions(screwdriverBuildJobFromSlime(deployOptions.field("screwdriverBuildJob")), optional("vespaVersion", deployOptions).map(Version::new), deployOptions.field("ignoreValidationErrors").asBool(), deployOptions.field("deployCurrentVersion").asBool()); - ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get("applicationZip")); controller.applications().validate(applicationPackage.deploymentSpec()); ActivateResult result = controller.applications().deployApplication(applicationId, zone, @@ -824,7 +821,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region) { Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); - Zone zone = new Zone(Environment.from(environment), RegionName.from(region)); + ZoneId zone = ZoneId.from(environment, region); Deployment deployment = application.deployments().get(zone); if (deployment == null) { // Attempt to deactivate application even if the deployment is not known by the controller @@ -873,8 +870,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } } - private Optional<String> userFrom(HttpRequest request) { - return authorizer.getPrincipalIfAny(request).map(Principal::getName); + private Optional<UserId> userFrom(HttpRequest request) { + return authorizer.getPrincipalIfAny(request) + .map(AthenzPrincipal::getIdentity) + .filter(AthenzUser.class::isInstance) + .map(AthenzUser.class::cast) + .map(AthenzUser::getUserId); } private void toSlime(Cursor object, Tenant tenant, HttpRequest request, boolean listApplications) { @@ -985,18 +986,22 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private void throwIfNotSuperUserOrPartOfOpsDbGroup(UserGroup userGroup, HttpRequest request) { - UserId userId = authorizer.getUserId(request); - if (!authorizer.isSuperUser(request) && !authorizer.isGroupMember(userId, userGroup) ) { + AthenzIdentity identity = authorizer.getIdentity(request); + if (!(identity instanceof AthenzUser)) { + throw new ForbiddenException("Identity not an user: " + identity.getFullName()); + } + AthenzUser user = (AthenzUser) identity; + if (!authorizer.isSuperUser(request) && !authorizer.isGroupMember(user.getUserId(), userGroup) ) { throw new ForbiddenException(String.format("User '%s' is not super user or part of the OpsDB user group '%s'", - userId.id(), userGroup.id())); + user.getUserId().id(), userGroup.id())); } } private void throwIfNotAthenzDomainAdmin(AthenzDomain tenantDomain, HttpRequest request) { - UserId userId = authorizer.getUserId(request); - if ( ! authorizer.isAthenzDomainAdmin(userId, tenantDomain)) { + AthenzIdentity identity = authorizer.getIdentity(request); + if ( ! authorizer.isAthenzDomainAdmin(identity, tenantDomain)) { throw new ForbiddenException( - String.format("The user '%s' is not admin in Athenz domain '%s'", userId.id(), tenantDomain.id())); + String.format("The user '%s' is not admin in Athenz domain '%s'", identity.getFullName(), tenantDomain.id())); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java index 0c808e30c2a..b7080a763f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java @@ -10,16 +10,17 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; -import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzIdentity; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.NToken; import com.yahoo.vespa.hosted.controller.common.ContextAttributes; import com.yahoo.vespa.hosted.controller.restapi.filter.NTokenRequestFilter; import javax.ws.rs.ForbiddenException; import javax.ws.rs.HttpMethod; import javax.ws.rs.core.SecurityContext; -import java.security.Principal; import java.util.Optional; import java.util.logging.Logger; @@ -54,27 +55,26 @@ public class Authorizer { Optional<Tenant> tenant = controller.tenants().tenant(tenantId); if ( ! tenant.isPresent()) return; - UserId userId = getUserId(request); - if (isTenantAdmin(userId, tenant.get())) return; + AthenzIdentity identity = getIdentity(request); + if (isTenantAdmin(identity, tenant.get())) return; - throw loggedForbiddenException("User " + userId + " does not have write access to tenant " + tenantId); + throw loggedForbiddenException("User " + identity.getFullName() + " does not have write access to tenant " + tenantId); } - public UserId getUserId(HttpRequest request) { - String name = getPrincipal(request).getName(); - if (name == null) - throw loggedForbiddenException("Not authorized: User name is null"); - return new UserId(name); + public AthenzIdentity getIdentity(HttpRequest request) { + return getPrincipal(request).getIdentity(); } /** Returns the principal or throws forbidden */ // TODO: Avoid REST exceptions - public Principal getPrincipal(HttpRequest request) { + public AthenzPrincipal getPrincipal(HttpRequest request) { return getPrincipalIfAny(request).orElseThrow(() -> Authorizer.loggedForbiddenException("User is not authenticated")); } /** Returns the principal if there is any */ - public Optional<Principal> getPrincipalIfAny(HttpRequest request) { - return securityContextOf(request).map(SecurityContext::getUserPrincipal); + public Optional<AthenzPrincipal> getPrincipalIfAny(HttpRequest request) { + return securityContextOf(request) + .map(SecurityContext::getUserPrincipal) + .map(AthenzPrincipal.class::cast); } public Optional<NToken> getNToken(HttpRequest request) { @@ -93,26 +93,36 @@ public class Authorizer { return new ForbiddenException(formattedMessage); } - private boolean isTenantAdmin(UserId userId, Tenant tenant) { + private boolean isTenantAdmin(AthenzIdentity identity, Tenant tenant) { switch (tenant.tenantType()) { case ATHENS: - return isAthenzTenantAdmin(userId, tenant.getAthensDomain().get()); - case OPSDB: - return isGroupMember(userId, tenant.getUserGroup().get()); - case USER: - return isUserTenantOwner(tenant.getId(), userId); + return isAthenzTenantAdmin(identity, tenant.getAthensDomain().get()); + case OPSDB: { + if (!(identity instanceof AthenzUser)) { + return false; + } + AthenzUser user = (AthenzUser) identity; + return isGroupMember(user.getUserId(), tenant.getUserGroup().get()); + } + case USER: { + if (!(identity instanceof AthenzUser)) { + return false; + } + AthenzUser user = (AthenzUser) identity; + return isUserTenantOwner(tenant.getId(), user.getUserId()); + } } throw new IllegalArgumentException("Unknown tenant type: " + tenant.tenantType()); } - private boolean isAthenzTenantAdmin(UserId userId, AthenzDomain tenantDomain) { + private boolean isAthenzTenantAdmin(AthenzIdentity athenzIdentity, AthenzDomain tenantDomain) { return athenzClientFactory.createZmsClientWithServicePrincipal() - .hasTenantAdminAccess(AthenzUtils.createPrincipal(userId), tenantDomain); + .hasTenantAdminAccess(athenzIdentity, tenantDomain); } - public boolean isAthenzDomainAdmin(UserId userId, AthenzDomain tenantDomain) { + public boolean isAthenzDomainAdmin(AthenzIdentity identity, AthenzDomain tenantDomain) { return athenzClientFactory.createZmsClientWithServicePrincipal() - .isDomainAdmin(AthenzUtils.createPrincipal(userId), tenantDomain); + .isDomainAdmin(identity, tenantDomain); } public boolean isGroupMember(UserId userId, UserGroup userGroup) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java index 71126259417..c7e03048ec8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java @@ -6,15 +6,17 @@ import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; -import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; -import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; -import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsException; import javax.ws.rs.ForbiddenException; import javax.ws.rs.NotAuthorizedException; import java.security.Principal; +import java.util.Objects; import java.util.logging.Logger; import static com.yahoo.vespa.hosted.controller.restapi.application.Authorizer.environmentRequiresAuthorization; @@ -38,7 +40,21 @@ public class DeployAuthorizer { public void throwIfUnauthorizedForDeploy(Principal principal, Environment environment, Tenant tenant, - ApplicationId applicationId) { + ApplicationId applicationId, + ApplicationPackage applicationPackage) { + // Validate that domain in identity configuration (deployment.xml) is same as tenant domain + applicationPackage.deploymentSpec().athenzDomain().ifPresent(identityDomain -> { + AthenzDomain tenantDomain = tenant.getAthensDomain().orElseThrow(() -> new IllegalArgumentException("Identity provider only available to Athenz onboarded tenants")); + if (! Objects.equals(tenantDomain.id(), identityDomain.value())) { + throw new ForbiddenException( + String.format( + "Athenz domain in deployment.xml: [%s] must match tenant domain: [%s]", + identityDomain.value(), + tenantDomain.id() + )); + } + }); + if (!environmentRequiresAuthorization(environment)) { return; } @@ -70,7 +86,7 @@ public class DeployAuthorizer { "Screwdriver principal '%1$s' does not have deploy access to '%2$s'. " + "Either the application has not been created at " + zoneRegistry.getDashboardUri() + " or " + "'%1$s' is not added to the application's deployer role in Athenz domain '%3$s'.", - athenzPrincipal.toYRN(), applicationId, tenantDomain.id()); + athenzPrincipal.getIdentity().getFullName(), applicationId, tenantDomain.id()); } } } @@ -91,7 +107,7 @@ public class DeployAuthorizer { try { return athenzClientFactory.createZmsClientWithServicePrincipal() .hasApplicationAccess( - principal, + principal.getIdentity(), ApplicationAction.deploy, domain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(applicationId.application().value())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java index 6a448e475c5..0b0a2c3ad52 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ServiceApiResponse.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; @@ -30,7 +30,7 @@ import java.util.regex.Pattern; */ class ServiceApiResponse extends HttpResponse { - private final Zone zone; + private final ZoneId zone; private final ApplicationId application; private final List<URI> configServerURIs; private final Slime slime; @@ -40,7 +40,7 @@ class ServiceApiResponse extends HttpResponse { private String serviceName = null; private String restPath = null; - public ServiceApiResponse(Zone zone, ApplicationId application, List<URI> configServerURIs, URI requestUri) { + public ServiceApiResponse(ZoneId zone, ApplicationId application, List<URI> configServerURIs, URI requestUri) { super(200); this.zone = zone; this.application = application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java index e350b98adb9..4d4f01bc1a6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/screwdriver/ScrewdriverApiHandler.java @@ -12,9 +12,7 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; @@ -109,7 +107,7 @@ public class ScrewdriverApiHandler extends LoggingRequestHandler { .orElse(JobType.component); ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); - controller.applications().lockedOrThrow(applicationId, application -> { + controller.applications().lockOrThrow(applicationId, application -> { // Since this is a manual operation we likely want it to trigger as soon as possible so we add it at to the // front of the queue application = controller.applications().deploymentTrigger().triggerAllowParallel( diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index 3a3fd445bcf..aecd3847653 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -70,7 +70,7 @@ public class ZoneApiHandler extends LoggingRequestHandler { private HttpResponse root(HttpRequest request) { List<Environment> environments = zoneRegistry.zones().stream() - .map(Zone::environment) + .map(ZoneId::environment) .distinct() .sorted(Comparator.comparing(Environment::value)) .collect(Collectors.toList()); @@ -89,7 +89,7 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse environment(HttpRequest request, Environment environment) { - List<Zone> zones = zoneRegistry.zones().stream() + List<ZoneId> zones = zoneRegistry.zones().stream() .filter(zone -> zone.environment() == environment) .collect(Collectors.toList()); Slime slime = new Slime(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 529b2b25785..3f85b0116ad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v2; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; @@ -81,7 +81,7 @@ public class ZoneApiHandler extends LoggingRequestHandler { } Environment environment = Environment.from(path.get("environment")); RegionName region = RegionName.from(path.get("region")); - Optional<Zone> zone = zoneRegistry.getZone(environment, region); + Optional<ZoneId> zone = zoneRegistry.getZone(environment, region); if (!zone.isPresent()) { throw new IllegalArgumentException("No such zone: " + environment.value() + "." + region.value()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java new file mode 100644 index 00000000000..a638756a600 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java @@ -0,0 +1,49 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.rotation; + +import java.util.Objects; + +/** + * Represents a global routing rotation. + * + * @author mpolden + */ +public class Rotation { + + private final RotationId id; + private final String name; + + public Rotation(RotationId id, String name) { + this.id = Objects.requireNonNull(id); + this.name = Objects.requireNonNull(name); + } + + /** The ID of the allocated rotation. This value is generated by global routing system */ + public RotationId id() { + return id; + } + + /** The global rotation FQDN */ + public String name() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Rotation)) return false; + final Rotation rotation = (Rotation) o; + return id().equals(rotation.id()) && name().equals(rotation.name()); + } + + @Override + public int hashCode() { + return Objects.hash(id(), name()); + } + + @Override + public String toString() { + return String.format("rotation %s -> %s", id().asString(), name()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java new file mode 100644 index 00000000000..10b15488f6e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java @@ -0,0 +1,42 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.rotation; + +import java.util.Objects; + +/** + * ID of a global rotation. + * + * @author mpolden + */ +public class RotationId { + + private final String id; + + public RotationId(String id) { + this.id = id; + } + + /** Rotation ID, e.g. rotation-42.vespa.global.routing */ + public String asString() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RotationId that = (RotationId) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + return "rotation ID " + id; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java new file mode 100644 index 00000000000..508df263837 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java @@ -0,0 +1,25 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.rotation; + +import com.yahoo.vespa.curator.Lock; + +import java.util.Objects; + +/** + * A lock for the rotation repository. This is a type-safe wrapper for a curator lock. + * + * @author mpolden + */ +public class RotationLock implements AutoCloseable { + + private final Lock lock; + + RotationLock(Lock lock) { + this.lock = Objects.requireNonNull(lock, "lock cannot be null"); + } + + @Override + public void close() { + lock.close(); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java new file mode 100644 index 00000000000..c0d3fd4758e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -0,0 +1,117 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.rotation; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; + +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.collectingAndThen; + +/** + * The rotation repository offers global rotations to Vespa applications. + * + * The list of rotations comes from RotationsConfig, which is set in the controller's services.xml. + * + * @author Oyvind Gronnesby + * @author mpolden + */ +public class RotationRepository { + + private static final Logger log = Logger.getLogger(RotationRepository.class.getName()); + + private final Map<RotationId, Rotation> allRotations; + private final ApplicationController applications; + private final CuratorDb curator; + + public RotationRepository(RotationsConfig rotationsConfig, ApplicationController applications, CuratorDb curator) { + this.allRotations = from(rotationsConfig); + this.applications = applications; + this.curator = curator; + } + + /** Acquire a exclusive lock for this */ + public RotationLock lock() { + return new RotationLock(curator.lockRotations()); + } + + /** + * Returns a rotation for the given application + * + * If a rotation is already assigned to the application, that rotation will be returned. + * If no rotation is assigned, return an available rotation. The caller is responsible for assigning the rotation. + * + * @param application The application requesting a rotation + * @param lock Lock which must be acquired by the caller + */ + public Rotation getRotation(Application application, RotationLock lock) { + if (application.rotation().isPresent()) { + return allRotations.get(application.rotation().get().id()); + } + if (!application.deploymentSpec().globalServiceId().isPresent()) { + throw new IllegalArgumentException("global-service-id is not set in deployment spec"); + } + long productionZones = application.deploymentSpec().zones().stream() + .filter(zone -> zone.deploysTo(Environment.prod)) + // Global rotations don't work for nodes in corp network + .filter(zone -> !isCorp(zone)) + .count(); + if (productionZones < 2) { + throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined"); + } + return findAvailableRotation(application, lock); + } + + /** + * Returns all unassigned rotations + * @param lock Lock which must be acquired by the caller + */ + public Map<RotationId, Rotation> availableRotations(@SuppressWarnings("unused") RotationLock lock) { + List<RotationId> assignedRotations = applications.asList().stream() + .filter(application -> application.rotation().isPresent()) + .map(application -> application.rotation().get().id()) + .collect(Collectors.toList()); + Map<RotationId, Rotation> unassignedRotations = new LinkedHashMap<>(this.allRotations); + assignedRotations.forEach(unassignedRotations::remove); + return Collections.unmodifiableMap(unassignedRotations); + } + + private Rotation findAvailableRotation(Application application, RotationLock lock) { + Map<RotationId, Rotation> availableRotations = availableRotations(lock); + if (availableRotations.isEmpty()) { + throw new IllegalStateException("Unable to assign global rotation to " + application.id() + + " - no rotations available"); + } + // Return first available rotation + RotationId rotation = availableRotations.keySet().iterator().next(); + log.info(String.format("Offering %s to application %s", rotation, application.id())); + return allRotations.get(rotation); + } + + private static boolean isCorp(DeploymentSpec.DeclaredZone zone) { + return zone.region().isPresent() && zone.region().get().value().contains("corp"); + } + + /** Returns a immutable map of rotation ID to rotation sorted by rotation ID */ + private static Map<RotationId, Rotation> from(RotationsConfig rotationConfig) { + return rotationConfig.rotations().entrySet().stream() + .map(entry -> new Rotation(new RotationId(entry.getKey()), entry.getValue().trim())) + .sorted(Comparator.comparing(rotation -> rotation.id().asString())) + .collect(collectingAndThen(Collectors.toMap(Rotation::id, + rotation -> rotation, + (k, v) -> v, + LinkedHashMap::new), + Collections::unmodifiableMap)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index d152cf80472..0e07f7b7589 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.component.Vtag; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; @@ -119,15 +120,16 @@ public class VersionStatus { private static ListMap<Version, String> findConfigServerVersions(Controller controller) { List<URI> configServers = controller.zoneRegistry().zones().stream() - .flatMap(zone -> controller.getConfigServerUris(zone.environment(), zone.region()).stream()) - .collect(Collectors.toList()); + .filter(zone -> ! zone.region().equals(RegionName.from("us-east-2a"))) + .flatMap(zone -> controller.getConfigServerUris(zone.environment(), zone.region()).stream()) + .collect(Collectors.toList()); ListMap<Version, String> versions = new ListMap<>(); for (URI configServer : configServers) versions.put(controller.applications().configserverClient().version(configServer), configServer.getHost()); return versions; } - + private static Collection<DeploymentStatistics> computeDeploymentStatistics(Set<Version> infrastructureVersions, List<Application> applications, Instant jobTimeoutLimit) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java deleted file mode 100644 index 363a2ea19cd..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/ControllerRotationRepository.java +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.rotation; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.hosted.controller.api.ApplicationAlias; -import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; -import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; -import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import org.jetbrains.annotations.NotNull; - -import java.net.URI; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * A rotation repository. - * - * @author Oyvind Gronnesby - */ -// TODO: Fold this into ApplicationController+Application -public class ControllerRotationRepository implements RotationRepository { - - private static final Logger log = Logger.getLogger(ControllerRotationRepository.class.getName()); - public static final String REMAINING_ROTATIONS_METRIC_NAME = "remaining_rotations"; - - private final ControllerDb controllerDb; - private final Map<RotationId, Rotation> rotationsMap; - private final Metric metric; - - public ControllerRotationRepository(RotationsConfig rotationConfig, ControllerDb controllerDb, Metric metric) { - this.controllerDb = controllerDb; - this.rotationsMap = buildRotationsMap(rotationConfig); - this.metric = metric; - } - - private static Map<RotationId, Rotation> buildRotationsMap(RotationsConfig rotationConfig) { - return rotationConfig.rotations().entrySet().stream() - .map(entry -> { - RotationId rotationId = new RotationId(entry.getKey()); - return new Rotation(rotationId, entry.getValue().trim()); - }) - .collect(Collectors.toMap( - rotation -> rotation.rotationId, - rotation -> rotation - )); - } - - @Override - @NotNull - public Set<Rotation> getOrAssignRotation(ApplicationId applicationId, DeploymentSpec deploymentSpec) { - reportRemainingRotations(); - - Set<RotationId> rotations = controllerDb.getRotations(applicationId); - - if (rotations.size() > 1) { - log.warning(String.format("Application %s has %d > 1 rotation", applicationId, rotations.size())); - } - - if (!rotations.isEmpty()) { - return rotations.stream() - .map(rotationsMap::get) - .collect(Collectors.toSet()); - } - - if (!deploymentSpec.globalServiceId().isPresent()) { - return Collections.emptySet(); - } - - long productionZoneCount = deploymentSpec.zones().stream() - .filter(zone -> zone.deploysTo(Environment.prod)) - .filter(zone -> ! isCorp(zone)) // Global rotations don't work for nodes in corp network - .count(); - - if (productionZoneCount >= 2) { - return assignRotation(applicationId); - } else { - throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined"); - } - } - - private static boolean isCorp(DeploymentSpec.DeclaredZone zone) { - return zone.region().isPresent() && zone.region().get().value().contains("corp"); - } - - @Override - @NotNull - public Set<URI> getRotationUris(ApplicationId applicationId) { - Set<RotationId> rotations = controllerDb.getRotations(applicationId); - if (rotations.isEmpty()) { - return Collections.emptySet(); - } - else { - ApplicationAlias applicationAlias = new ApplicationAlias(applicationId); - return Collections.singleton(applicationAlias.toHttpUri()); - } - } - - private Set<Rotation> assignRotation(ApplicationId applicationId) { - Set<RotationId> availableRotations = availableRotations(); - if (availableRotations.isEmpty()) { - String message = "Unable to assign global rotation to " - + applicationId + " - no rotations available"; - log.info(message); - throw new RuntimeException(message); - } - - for (RotationId rotationId : availableRotations) { - if (controllerDb.assignRotation(rotationId, applicationId)) { - log.info(String.format("Assigned rotation %s to application %s", rotationId, applicationId)); - Rotation rotation = this.rotationsMap.get(rotationId); - return Collections.singleton(rotation); - } - } - - log.info(String.format("Rotation: No rotations assigned with %s rotations available", availableRotations.size())); - return Collections.emptySet(); - } - - private Set<RotationId> availableRotations() { - Set<RotationId> assignedRotations = controllerDb.getRotations(); - Set<RotationId> allRotations = new HashSet<>(rotationsMap.keySet()); - allRotations.removeAll(assignedRotations); - return allRotations; - } - - private void reportRemainingRotations() { - try { - int freeRotationsCount = availableRotations().size(); - log.log(LogLevel.INFO, "Rotation: {0} global rotations remaining", freeRotationsCount); - metric.set(REMAINING_ROTATIONS_METRIC_NAME, freeRotationsCount, - metric.createContext(Collections.emptyMap())); - } catch (Exception e) { - log.log(LogLevel.INFO, "Failed to report rotations metric", e); - } - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java deleted file mode 100644 index 4e333f0268b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/MemoryRotationRepository.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.rotation; - -import com.google.common.collect.ImmutableSet; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.identifiers.RotationId; -import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; -import org.jetbrains.annotations.NotNull; - -import java.net.URI; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; - -/** - * A rotation repository backed by in-memory data structures - * - * @author bratseth - */ -public class MemoryRotationRepository implements RotationRepository { - - private final Map<ApplicationId, Set<Rotation>> rotations = new HashMap<>(); - - @NotNull - @Override - public Set<Rotation> getOrAssignRotation(ApplicationId application, DeploymentSpec deploymentSpec) { - if (rotations.containsKey(application)) { - return rotations.get(application); - } - Set<Rotation> rotations = ImmutableSet.of(new Rotation( - new RotationId("generated-by-routing-service-" + UUID.randomUUID().toString()), - "fake-global-rotation-" + application.toShortString()) - ); - this.rotations.put(application, rotations); - return rotations; - } - - @NotNull - @Override - public Set<URI> getRotationUris(ApplicationId applicationId) { - Set<Rotation> rotations = this.rotations.get(applicationId); - if (rotations == null) { - return Collections.emptySet(); - } - return rotations.stream() - .map(rotation -> URI.create("http://" + rotation.rotationName)) - .collect(Collectors.toSet()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java deleted file mode 100644 index b1f7b33e58e..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/rotation/RotationRepository.java +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.rotation; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; -import org.jetbrains.annotations.NotNull; - -import java.net.URI; -import java.util.Set; - -/** - * A rotation repository assigns global rotations to Vespa applications. It does not take into account - * whether an application qualifies or not, but it assumes that each application should get only - * one. - * - * The list of rotations comes from the RotationsConfig, set in the controller's services.xml. - * Assignments are persisted with the RotationId as the primary key. When we assign the - * rotation to an application we try to put the mapping RotationId -> Application. If a - * mapping already exists for that RotationId, the assignment will fail. - * - * @author Oyvind Gronnesby - */ -public interface RotationRepository { - - // TODO: Change to use provision.ApplicationId - // TODO: Move the persistence into ControllerDb (done), and then collapse the 2 implementations and the interface into one - - /** - * If any rotations are assigned to the application, these will be returned. - * If no rotations are assigned, assign one rotation to the application and return that. - * - * @param applicationId ID of the application to get or assign rotation for - * @param deploymentSpec Spec of current application being deployed - * @return Set of rotations assigned (may be empty) - */ - @NotNull - Set<Rotation> getOrAssignRotation(ApplicationId applicationId, DeploymentSpec deploymentSpec); - - /** - * Get the external visible rotation URIs for this application. - * - * @param applicationId ID of the application to get or assign rotation for - */ - @NotNull - Set<URI> getRotationUris(ApplicationId applicationId); - -} |