diff options
Diffstat (limited to 'controller-server/src')
69 files changed, 2565 insertions, 351 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 672f50f83d7..7512e14643f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -22,10 +22,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; @@ -44,6 +40,10 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.maintenance.DeploymentExpirer; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; @@ -84,7 +84,7 @@ public class ApplicationController { private final CuratorDb curator; private final RotationRepository rotationRepository; - private final ZmsClientFactory zmsClientFactory; + private final AthenzClientFactory zmsClientFactory; private final NameService nameService; private final ConfigServerClient configserverClient; private final RoutingGenerator routingGenerator; @@ -94,7 +94,7 @@ public class ApplicationController { ApplicationController(Controller controller, ControllerDb db, CuratorDb curator, RotationRepository rotationRepository, - ZmsClientFactory zmsClientFactory, + AthenzClientFactory zmsClientFactory, NameService nameService, ConfigServerClient configserverClient, RoutingGenerator routingGenerator, Clock clock) { this.controller = controller; @@ -249,7 +249,7 @@ public class ApplicationController { if (tenant.get().isAthensTenant() && ! token.isPresent()) throw new IllegalArgumentException("Could not create '" + id + "': No NToken provided"); if (tenant.get().isAthensTenant()) { - ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()); + ZmsClient zmsClient = zmsClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()); try { zmsClient.deleteApplication(tenant.get().getAthensDomain().get(), new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); @@ -331,13 +331,10 @@ public class ApplicationController { applicationPackage.zippedContent()); preparedApplication.activate(); - Deployment previousDeployment = application.deployments().get(zone); - Deployment newDeployment = previousDeployment; - if (previousDeployment == null) { - newDeployment = new Deployment(zone, revision, version, clock.instant(), new HashMap<>(), new HashMap<>()); - } else { - newDeployment = new Deployment(zone, revision, version, clock.instant(), previousDeployment.clusterUtils(), previousDeployment.clusterInfo()); - } + // Use info from previous deployments is available + Deployment previousDeployment = application.deployments().getOrDefault(zone, new Deployment(zone, revision, version, clock.instant())); + Deployment newDeployment = new Deployment(zone, revision, version, clock.instant(), + previousDeployment.clusterUtils(), previousDeployment.clusterInfo(), previousDeployment.metrics()); application = application.with(newDeployment); store(application, lock); @@ -493,7 +490,7 @@ public class ApplicationController { // NB: Next 2 lines should have been one transaction if (tenant.isAthensTenant()) - zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()) + zmsClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()) .deleteApplication(tenant.getAthensDomain().get(), new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(id.application().value())); db.deleteApplication(id); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 4f5c2ead4da..e338bc17788 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; +import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; @@ -14,8 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; -import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; @@ -26,6 +25,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingSe import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; @@ -40,7 +41,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; /** @@ -64,15 +64,7 @@ public class Controller extends AbstractComponent { private final CuratorDb curator; private final ApplicationController applicationController; private final TenantController tenantController; - - /** - * Status of Vespa versions across the system. - * This is expensive to maintain so that is done periodically by a maintenance job - */ - private final AtomicReference<VersionStatus> versionStatus; - private final Clock clock; - private final RotationRepository rotationRepository; private final GitHub gitHub; private final EntityService entityService; @@ -81,7 +73,6 @@ public class Controller extends AbstractComponent { private final ConfigServerClient configServerClient; private final MetricsService metricsService; private final Chef chefClient; - private final Athens athens; private final ZmsClient zmsClient; /** @@ -96,11 +87,11 @@ public class Controller extends AbstractComponent { GlobalRoutingService globalRoutingService, ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, MetricsService metricsService, NameService nameService, - RoutingGenerator routingGenerator, Chef chefClient, Athens athens) { + RoutingGenerator routingGenerator, Chef chefClient, AthenzClientFactory athenzClientFactory) { this(db, curator, rotationRepository, gitHub, jiraClient, entityService, globalRoutingService, zoneRegistry, configServerClient, metricsService, nameService, routingGenerator, chefClient, - Clock.systemUTC(), athens); + Clock.systemUTC(), athenzClientFactory); } public Controller(ControllerDb db, CuratorDb curator, RotationRepository rotationRepository, @@ -108,7 +99,8 @@ public class Controller extends AbstractComponent { GlobalRoutingService globalRoutingService, ZoneRegistry zoneRegistry, ConfigServerClient configServerClient, MetricsService metricsService, NameService nameService, - RoutingGenerator routingGenerator, Chef chefClient, Clock clock, Athens athens) { + RoutingGenerator routingGenerator, Chef chefClient, Clock clock, + AthenzClientFactory athenzClientFactory) { Objects.requireNonNull(db, "Controller db cannot be null"); Objects.requireNonNull(curator, "Curator cannot be null"); Objects.requireNonNull(rotationRepository, "Rotation repository cannot be null"); @@ -123,7 +115,7 @@ public class Controller extends AbstractComponent { Objects.requireNonNull(routingGenerator, "RoutingGenerator cannot be null"); Objects.requireNonNull(chefClient, "ChefClient cannot be null"); Objects.requireNonNull(clock, "Clock cannot be null"); - Objects.requireNonNull(athens, "Athens cannot be null"); + Objects.requireNonNull(athenzClientFactory, "Athens cannot be null"); this.rotationRepository = rotationRepository; this.curator = curator; @@ -135,13 +127,11 @@ public class Controller extends AbstractComponent { this.metricsService = metricsService; this.chefClient = chefClient; this.clock = clock; - this.athens = athens; - this.zmsClient = athens.zmsClientFactory().createClientWithServicePrincipal(); + this.zmsClient = athenzClientFactory.createZmsClientWithServicePrincipal(); - applicationController = new ApplicationController(this, db, curator, rotationRepository, athens.zmsClientFactory(), + applicationController = new ApplicationController(this, db, curator, rotationRepository, athenzClientFactory, nameService, configServerClient, routingGenerator, clock); - tenantController = new TenantController(this, db, curator, entityService); - versionStatus = new AtomicReference<>(VersionStatus.empty()); + tenantController = new TenantController(this, db, curator, entityService, athenzClientFactory); } /** Returns the instance controlling tenants */ @@ -154,10 +144,6 @@ public class Controller extends AbstractComponent { return zmsClient.getDomainList(prefix); } - public Athens athens() { - return athens; - } - /** * Fetch list of all active OpsDB properties. * @@ -191,11 +177,7 @@ public class Controller extends AbstractComponent { "sort:!('@timestamp',desc))"; URI kibanaPath = URI.create(kibanaQuery); - if (kibanaHost.isPresent()) { - return kibanaHost.get().resolve(kibanaPath); - } else { - return null; - } + return kibanaHost.map(uri -> uri.resolve(kibanaPath)).orElse(null); } public Set<URI> getRotationUris(ApplicationId id) { @@ -235,17 +217,19 @@ public class Controller extends AbstractComponent { ! newStatus.systemVersion().equals(currentStatus.systemVersion())) { log.info("Changing system version from " + printableVersion(currentStatus.systemVersion()) + " to " + printableVersion(newStatus.systemVersion())); - curator.writeSystemVersion(newStatus.systemVersion().get().versionNumber()); } - - this.versionStatus.set(newStatus); + curator.writeVersionStatus(newStatus); } /** Returns the latest known version status. Calling this is free but the status may be slightly out of date. */ - public VersionStatus versionStatus() { return versionStatus.get(); } + public VersionStatus versionStatus() { return curator.readVersionStatus(); } /** Returns the current system version: The controller should drive towards running all applications on this version */ - public Version systemVersion() { return curator.readSystemVersion(); } + public Version systemVersion() { + return versionStatus().systemVersion() + .map(VespaVersion::versionNumber) + .orElse(Vtag.currentVersion); + } public MetricsService metricsService() { return metricsService; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java index fafd0b04dd2..229c46f0a22 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/TenantController.java @@ -11,11 +11,12 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClient; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.PersistenceException; @@ -47,14 +48,15 @@ public class TenantController { /** For working memory storage and sharing between controllers */ private final CuratorDb curator; - private final ZmsClientFactory zmsClientFactory; + private final AthenzClientFactory athenzClientFactory; private final EntityService entityService; - public TenantController(Controller controller, ControllerDb db, CuratorDb curator, EntityService entityService) { + public TenantController(Controller controller, ControllerDb db, CuratorDb curator, EntityService entityService, + AthenzClientFactory athenzClientFactory) { this.controller = controller; this.db = db; this.curator = curator; - this.zmsClientFactory = controller.athens().zmsClientFactory(); + this.athenzClientFactory = athenzClientFactory; this.entityService = entityService; } @@ -64,8 +66,8 @@ public class TenantController { public List<Tenant> asList(UserId user) { Set<UserGroup> userGroups = entityService.getUserGroups(user); - Set<AthensDomain> userDomains = new HashSet<>(zmsClientFactory.createClientWithServicePrincipal() - .getTenantDomainsForUser(controller.athens().principalFrom(user))); + Set<AthensDomain> userDomains = new HashSet<>(athenzClientFactory.createZtsClientWithServicePrincipal() + .getTenantDomainsForUser(AthenzUtils.createPrincipal(user))); Predicate<Tenant> hasUsersGroup = (tenant) -> tenant.getUserGroup().isPresent() && userGroups.contains(tenant.getUserGroup().get()); Predicate<Tenant> hasUsersDomain = (tenant) -> tenant.getAthensDomain().isPresent() && userDomains.contains(tenant.getAthensDomain().get()); @@ -108,7 +110,7 @@ public class TenantController { if (existingTenantWithDomain.isPresent()) throw new IllegalArgumentException("Could not create " + tenant + ": The Athens domain '" + domain + "' is already connected to " + existingTenantWithDomain.get()); - ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()); + ZmsClient zmsClient = athenzClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()); try { zmsClient.deleteTenant(domain); } catch (ZmsException ignored) { } zmsClient.createTenant(domain); } @@ -158,7 +160,7 @@ public class TenantController { throw new IllegalArgumentException("Could not set domain of " + updatedTenant + " to '" + newDomain + "':" + existingTenantWithNewDomain.get() + " already has this domain"); - ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()); + ZmsClient zmsClient = athenzClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()); zmsClient.createTenant(newDomain); List<Application> applications = controller.applications().asList(TenantName.from(existingTenant.getId().id())); applications.forEach(a -> zmsClient.addApplication(newDomain, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId(a.id().application().value()))); @@ -184,7 +186,8 @@ public class TenantController { throw new RuntimeException(e); } if (tenant.isAthensTenant()) - zmsClientFactory.createClientWithAuthorizedServiceToken(token.get()).deleteTenant(tenant.getAthensDomain().get()); + athenzClientFactory.createZmsClientWithAuthorizedServiceToken(token.get()) + .deleteTenant(tenant.getAthensDomain().get()); log.info("Deleted " + tenant); } } @@ -204,7 +207,7 @@ public class TenantController { throw new IllegalArgumentException("Could not migrate " + existing + " to " + tenantDomain + ": " + "Tenant is not currently an OpsDb tenant"); - ZmsClient zmsClient = zmsClientFactory.createClientWithAuthorizedServiceToken(nToken); + ZmsClient zmsClient = athenzClientFactory.createZmsClientWithAuthorizedServiceToken(nToken); zmsClient.createTenant(tenantDomain); List<Application> applications = controller.applications().asList(TenantName.from(existing.getId().id())); applications.forEach(a -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java index 03d0cd28ca1..985d6173b29 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterCost.java @@ -6,13 +6,14 @@ package com.yahoo.vespa.hosted.controller.application; * tco and waste for one cluster of one deployment. * * The target utilization is defined the following assumptions: - * 1. CPU contention starts to cause problems on 0.8 - * 2. Memory management starts to casue problems on 0.7 + * 1. CPU contention starts to cause problems at 0.8 + * 2. Memory management starts to cause problems at 0.7 * 3. Load is evenly divided between two deployments - each deployments can handle the other. * 4. Memory and disk are agnostic to query load. * 5. Peak utilization (daily variations) are twice the size of the average. * * With this in mind we get: + * * CPU: 0.8/2/2 = 0.2 * Mem: 0.7 * Disk: 0.7 @@ -40,16 +41,18 @@ public class ClusterCost { this.targetUtilization = new ClusterUtilization(0.7,0.2, 0.7, 0.3); this.resultUtilization = calculateResultUtilization(systemUtilization, targetUtilization); - this.tco = clusterInfo.getCost() * Math.min(1, this.resultUtilization.getMaxUtilization()); - this.waste = clusterInfo.getCost() - tco; + this.tco = clusterInfo.getHostnames().size() * clusterInfo.getFlavorCost(); + + double unusedUtilization = 1 - Math.min(1, resultUtilization.getMaxUtilization()); + this.waste = tco * unusedUtilization; } - /** @return TCO in dollars */ + /** @return The TCO in dollars for this cluster (node tco * nodes) */ public double getTco() { return tco; } - /** @return Waste in dollars */ + /** @return The amount of dollars spent for unused resources in this cluster */ public double getWaste() { return waste; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java index cb39177c811..40fc57acdc8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ClusterInfo.java @@ -9,31 +9,62 @@ import java.util.List; * Value object of static cluster information, in particular the TCO * of the hardware used for this cluster. * + * Some duplication/flattening of flavor info is done to simplify client usage. + * * @author smorgrav */ public class ClusterInfo { private final String flavor; - private final int cost; + private final double flavorCPU; + private final double flavorMem; + private final double flavorDisk; + private final int flavorCost; private final ClusterSpec.Type clusterType; private final List<String> hostnames; - public ClusterInfo(String flavor, int cost, ClusterSpec.Type clusterType, List<String> hostnames) { + /** + * @param flavor The name of the flavor eg. 'C-2B/24/500' + * @param flavorCost The cost of one node in dollars + * @param flavorCPU The number of cpu cores granted + * @param flavorMem The memory granted in Gb + * @param flavorDisk The disk size granted in Gb + * @param clusterType The vespa cluster type e.g 'container' or 'content' + * @param hostnames All hostnames in this cluster + */ + public ClusterInfo(String flavor, int flavorCost, double flavorCPU, double flavorMem, + double flavorDisk, ClusterSpec.Type clusterType, List<String> hostnames) { this.flavor = flavor; - this.cost = cost; + this.flavorCost = flavorCost; + this.flavorCPU = flavorCPU; + this.flavorMem = flavorMem; + this.flavorDisk = flavorDisk; this.clusterType = clusterType; this.hostnames = hostnames; } + /** @return The name of the flavor eg. 'C-2B/24/500' */ public String getFlavor() { return flavor; } - public int getCost() { return cost; } + /** @return The cost of one node in dollars */ + public int getFlavorCost() { return flavorCost; } + + /** @return The disk size granted in Gb */ + public double getFlavorDisk() { return flavorDisk; } + + /** @return The number of cpu cores granted */ + public double getFlavorCPU() { return flavorCPU; } + + /** @return The memory granted in Gb */ + public double getFlavorMem() { return flavorMem; } + /** @return The vespa cluster type e.g 'container' or 'content' */ public ClusterSpec.Type getClusterType() { return clusterType; } + /** @return All hostnames in this cluster */ public List<String> getHostnames() { return hostnames; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java index 01219e940a3..05a7f9667ba 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Deployment.java @@ -14,6 +14,7 @@ import java.util.Objects; * A deployment of an application in a particular zone. * * @author bratseth + * @author smorgrav */ public class Deployment { @@ -23,24 +24,28 @@ public class Deployment { private final Instant deployTime; private final Map<Id, ClusterUtilization> clusterUtils; private final Map<Id, ClusterInfo> clusterInfo; + private final DeploymentMetrics metrics; public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime) { - this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>()); + this(zone, revision, version, deployTime, new HashMap<>(), new HashMap<>(), new DeploymentMetrics()); } - public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime, Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo) { + public Deployment(Zone zone, ApplicationRevision revision, Version version, Instant deployTime, + Map<Id, ClusterUtilization> clusterUtils, Map<Id, ClusterInfo> clusterInfo, DeploymentMetrics metrics) { Objects.requireNonNull(zone, "zone cannot be null"); Objects.requireNonNull(revision, "revision cannot be null"); Objects.requireNonNull(version, "version cannot be null"); Objects.requireNonNull(deployTime, "deployTime cannot be null"); Objects.requireNonNull(clusterUtils, "clusterUtils cannot be null"); Objects.requireNonNull(clusterInfo, "clusterInfo cannot be null"); + Objects.requireNonNull(clusterInfo, "deployment metrics cannot be null"); this.zone = zone; this.revision = revision; this.version = version; this.deployTime = deployTime; this.clusterUtils = clusterUtils; this.clusterInfo = clusterInfo; + this.metrics = metrics; } /** Returns the zone this was deployed to */ @@ -64,11 +69,20 @@ public class Deployment { } public Deployment withClusterUtils(Map<Id, ClusterUtilization> clusterUtilization) { - return new Deployment(zone, revision, version, deployTime, clusterUtilization, clusterInfo); + return new Deployment(zone, revision, version, deployTime, clusterUtilization, clusterInfo, metrics); } public Deployment withClusterInfo(Map<Id, ClusterInfo> newClusterInfo) { - return new Deployment(zone, revision, version, deployTime, clusterUtils, newClusterInfo); + return new Deployment(zone, revision, version, deployTime, clusterUtils, newClusterInfo, metrics); + } + + public Deployment withMetrics(DeploymentMetrics metrics) { + return new Deployment(zone, revision, version, deployTime, clusterUtils, clusterInfo, metrics); + } + + /** @return Key metrics for the deployment (application level) like QPS and document count */ + public DeploymentMetrics metrics() { + return metrics; } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java index fce825bd99e..585690793bb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentCost.java @@ -23,12 +23,16 @@ public class DeploymentCost { double tco = 0; double util = 0; double waste = 0; + double maxWaste = -1; for (ClusterCost costCluster : clusterCosts.values()) { tco += costCluster.getTco(); waste += costCluster.getWaste(); - int nodesInCluster = costCluster.getClusterInfo().getHostnames().size(); - util = Math.max(util, nodesInCluster*costCluster.getResultUtilization().getMaxUtilization()); + + if (costCluster.getWaste() > maxWaste) { + util = costCluster.getResultUtilization().getMaxUtilization(); + maxWaste = costCluster.getWaste(); + } } this.utilization = util; @@ -40,14 +44,17 @@ public class DeploymentCost { return clusters; } + /** @return Total cost of ownership for the deployment (sum of all clusters) */ public double getTco() { return tco; } + /** @return The utilization of clusters that wastes most money in this deployment */ public double getUtilization() { return utilization; } + /** @return The amount of dollars spent and not utilized */ public double getWaste() { return waste; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java new file mode 100644 index 00000000000..6812e4cb468 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentMetrics.java @@ -0,0 +1,50 @@ +package com.yahoo.vespa.hosted.controller.application;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @author smorgrav + */ +public class DeploymentMetrics { + + private final double queriesPerSecond; + private final double writesPerSecond; + private final double documentCount; + private final double queryLatencyMillis; + private final double writeLatencyMills; + + DeploymentMetrics() { + this.queriesPerSecond = 0; + this.writesPerSecond = 0; + this.documentCount = 0; + this.queryLatencyMillis = 0; + this.writeLatencyMills = 0; + } + + public DeploymentMetrics(double queriesPerSecond, double writesPerSecond, double documentCount, + double queryLatencyMillis, double writeLatencyMills) { + this.queriesPerSecond = queriesPerSecond; + this.writesPerSecond = writesPerSecond; + this.documentCount = documentCount; + this.queryLatencyMillis = queryLatencyMillis; + this.writeLatencyMills = writeLatencyMills; + } + + public double queriesPerSecond() { + return queriesPerSecond; + } + + public double writesPerSecond() { + return writesPerSecond; + } + + public double documentCount() { + return documentCount; + } + + public double queryLatencyMillis() { + return queryLatencyMillis; + } + + public double writeLatencyMillis() { + return writeLatencyMills; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java new file mode 100644 index 00000000000..8614414dc95 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ApplicationAction.java @@ -0,0 +1,17 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +/** + * @author bjorncs + */ +public enum ApplicationAction { + deploy("deployer"), + read("reader"), + write("writer"); + + public final String roleName; + + ApplicationAction(String roleName) { + this.roleName = roleName; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzClientFactory.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzClientFactory.java new file mode 100644 index 00000000000..b6a21f94f74 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzClientFactory.java @@ -0,0 +1,15 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +/** + * @author bjorncs + */ +public interface AthenzClientFactory { + + ZmsClient createZmsClientWithServicePrincipal(); + + ZtsClient createZtsClientWithServicePrincipal(); + + ZmsClient createZmsClientWithAuthorizedServiceToken(NToken authorizedServiceToken); + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPrincipal.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPrincipal.java new file mode 100644 index 00000000000..03d9f60c6b0 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPrincipal.java @@ -0,0 +1,62 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +import java.security.Principal; +import java.util.Objects; + +/** + * @author bjorncs + */ +public class AthenzPrincipal implements Principal { + + private final AthensDomain domain; + private final UserId userId; + + public AthenzPrincipal(AthensDomain domain, UserId userId) { + this.domain = domain; + this.userId = userId; + } + + public UserId getUserId() { + return userId; + } + + public AthensDomain getDomain() { + return domain; + } + + public String toYRN() { + return domain.id() + "." + userId.id(); + } + + @Override + public String getName() { + return userId.id(); + } + + @Override + public String toString() { + return "AthenzPrincipal{" + + "domain=" + domain + + ", userId=" + userId + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AthenzPrincipal that = (AthenzPrincipal) o; + return Objects.equals(domain, that.domain) && + Objects.equals(userId, that.userId); + } + + @Override + public int hashCode() { + return Objects.hash(domain, userId); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPublicKey.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPublicKey.java new file mode 100644 index 00000000000..01596ead0f4 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzPublicKey.java @@ -0,0 +1,49 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import java.security.PublicKey; +import java.util.Objects; + +/** + * @author bjorncs + */ +public class AthenzPublicKey { + + private final PublicKey publicKey; + private final String keyId; + + public AthenzPublicKey(PublicKey publicKey, String keyId) { + this.publicKey = publicKey; + this.keyId = keyId; + } + + public PublicKey getPublicKey() { + return publicKey; + } + + public String getKeyId() { + return keyId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AthenzPublicKey that = (AthenzPublicKey) o; + return Objects.equals(publicKey, that.publicKey) && + Objects.equals(keyId, that.keyId); + } + + @Override + public int hashCode() { + return Objects.hash(publicKey, keyId); + } + + @Override + public String toString() { + return "AthenzPublicKey{" + + "publicKey=" + publicKey + + ", keyId='" + keyId + '\'' + + '}'; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzService.java new file mode 100644 index 00000000000..780a14e4446 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzService.java @@ -0,0 +1,55 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; + +import java.util.Objects; + +/** + * @author bjorncs + */ +public class AthenzService { + + private final AthensDomain domain; + private final String serviceName; + + public AthenzService(AthensDomain domain, String serviceName) { + this.domain = domain; + this.serviceName = serviceName; + } + + public AthenzService(String domain, String serviceName) { + this(new AthensDomain(domain), serviceName); + } + + public String toFullServiceName() { + return domain.id() + "." + serviceName; + } + + public AthensDomain getDomain() { + return domain; + } + + public String getServiceName() { + return serviceName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AthenzService that = (AthenzService) o; + return Objects.equals(domain, that.domain) && + Objects.equals(serviceName, that.serviceName); + } + + @Override + public int hashCode() { + return Objects.hash(domain, serviceName); + } + + @Override + public String toString() { + return String.format("AthenzService(%s)", toFullServiceName()); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUtils.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUtils.java new file mode 100644 index 00000000000..0c0f4729100 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/AthenzUtils.java @@ -0,0 +1,29 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +/** + * @author bjorncs + */ +public class AthenzUtils { + + private AthenzUtils() {} + + // TODO Change to "user" as primary user principal domain. Also support "yby" for a limited time as per recent Athenz changes + public static final AthensDomain USER_PRINCIPAL_DOMAIN = new AthensDomain("yby"); + public static final AthensDomain SCREWDRIVER_DOMAIN = new AthensDomain("cd.screwdriver.project"); + public static final AthenzService ZMS_ATHENZ_SERVICE = new AthenzService("sys.auth", "zms"); + + public static AthenzPrincipal createPrincipal(UserId userId) { + return new AthenzPrincipal(USER_PRINCIPAL_DOMAIN, userId); + } + + public static AthenzPrincipal createPrincipal(ScrewdriverId screwdriverId) { + return new AthenzPrincipal(SCREWDRIVER_DOMAIN, new UserId("sd" + screwdriverId.id())); + } + + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/InvalidTokenException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/InvalidTokenException.java new file mode 100644 index 00000000000..e41bd8d4283 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/InvalidTokenException.java @@ -0,0 +1,11 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +/** + * @author bjorncs + */ +public class InvalidTokenException extends Exception { + public InvalidTokenException(String message) { + super(message); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/NToken.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/NToken.java new file mode 100644 index 00000000000..fec0523aaab --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/NToken.java @@ -0,0 +1,148 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import com.yahoo.athenz.auth.token.PrincipalToken; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; + +import java.security.PrivateKey; +import java.security.PublicKey; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; + +/** + * Represents an Athenz NToken (principal token) + * + * @author bjorncs + */ +// TODO Split out encoding/decoding of token into separate class. Move NToken to controller-api. +public class NToken { + + // Max allowed skew in token timestamp (only for creation, not expiry timestamp) + private static final int ALLOWED_TIMESTAMP_OFFSET = (int) TimeUnit.SECONDS.toSeconds(300); + + private final PrincipalToken token; + + // Note: PrincipalToken does not provide any way of constructing an instance from a unsigned token string + public NToken(String signedToken) { + try { + this.token = new PrincipalToken(signedToken); + if (this.token.getSignature() == null) { + throw new IllegalArgumentException("Signature missing (unsigned token)"); + } + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Malformed NToken: " + e.getMessage()); + } + } + + public AthenzPrincipal getPrincipal() { + return new AthenzPrincipal(getDomain(), getUser()); + } + + public UserId getUser() { + return new UserId(token.getName()); + } + + public AthensDomain getDomain() { + return new AthensDomain(token.getDomain()); + } + + public String getToken() { + return token.getSignedToken(); + } + + public String getKeyId() { + return token.getKeyId(); + } + + public void validateSignatureAndExpiration(PublicKey publicKey) throws InvalidTokenException { + StringBuilder errorMessageBuilder = new StringBuilder(); + if (!token.validate(publicKey, ALLOWED_TIMESTAMP_OFFSET, true, errorMessageBuilder)) { + throw new InvalidTokenException("NToken is expired or has invalid signature: " + errorMessageBuilder.toString()); + } + } + + @Override + public String toString() { + return String.format("NToken(%s)", getToken()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NToken nToken = (NToken) o; + return Objects.equals(getToken(), nToken.getToken()); // PrincipalToken does not implement equals() + } + + @Override + public int hashCode() { + return Objects.hash(getToken()); // PrincipalToken does not implement hashcode() + } + + public static class Builder { + + private final String version; + private final AthenzPrincipal principal; + private final PrivateKey privateKey; + private final String keyId; + private Optional<String> salt = Optional.empty(); + private Optional<String> hostname = Optional.empty(); + private Optional<String> ip = Optional.empty(); + private OptionalLong issueTime = OptionalLong.empty(); + private OptionalLong expirationWindow = OptionalLong.empty(); + + /** + * NOTE: We must have some signature, else we might end up with problems later on as + * {@link PrincipalToken#PrincipalToken(String)} only accepts signed token + * (supplying an unsigned token to the constructor will result in inconsistent state) + */ + public Builder(String version, AthenzPrincipal principal, PrivateKey privateKey, String keyId) { + this.version = version; + this.principal = principal; + this.privateKey = privateKey; + this.keyId = keyId; + } + + public Builder salt(String salt) { + this.salt = Optional.of(salt); + return this; + } + + public Builder hostname(String hostname) { + this.hostname = Optional.of(hostname); + return this; + } + + public Builder ip(String ip) { + this.ip = Optional.of(ip); + return this; + } + + public Builder issueTime(long issueTime) { + this.issueTime = OptionalLong.of(issueTime); + return this; + } + + public Builder expirationWindow(long expirationWindow) { + this.expirationWindow = OptionalLong.of(expirationWindow); + return this; + } + + public NToken build() { + PrincipalToken token = new PrincipalToken.Builder(version, principal.getDomain().id(), principal.getName()) + .keyId(this.keyId) + .salt(this.salt.orElse(null)) + .host(this.hostname.orElse(null)) + .ip(this.ip.orElse(null)) + .issueTime(this.issueTime.orElse(0)) + .expirationWindow(this.expirationWindow.orElse(0)) + .build(); + token.sign(this.privateKey); + return new NToken(token.getSignedToken()); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsClient.java new file mode 100644 index 00000000000..274a8fdf438 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsClient.java @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; + +import java.util.List; + +/** + * @author bjorncs + */ +public interface ZmsClient { + + void createTenant(AthensDomain tenantDomain); + + void deleteTenant(AthensDomain tenantDomain); + + void addApplication(AthensDomain tenantDomain, ApplicationId applicationName); + + void deleteApplication(AthensDomain tenantDomain, ApplicationId applicationName); + + boolean hasApplicationAccess(AthenzPrincipal principal, ApplicationAction action, AthensDomain tenantDomain, ApplicationId applicationName); + + boolean hasTenantAdminAccess(AthenzPrincipal principal, AthensDomain tenantDomain); + + // Used before vespa tenancy is established for the domain. + boolean isDomainAdmin(AthenzPrincipal principal, AthensDomain domain); + + List<AthensDomain> getDomainList(String prefix); + + AthenzPublicKey getPublicKey(AthenzService service, String keyId); + + List<AthenzPublicKey> getPublicKeys(AthenzService service); + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsException.java new file mode 100644 index 00000000000..59548339d11 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsException.java @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import com.yahoo.athenz.zms.ZMSClientException; + +/** + * @author bjorncs + */ +public class ZmsException extends RuntimeException { + + private final int code; + + public ZmsException(ZMSClientException e) { + super(e.getMessage(), e); + this.code = e.getCode(); + } + + + public int getCode() { + return code; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsKeystore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsKeystore.java new file mode 100644 index 00000000000..93fed95c768 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZmsKeystore.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import java.security.PublicKey; +import java.util.Optional; + +/** + * @author bjorncs + */ +public interface ZmsKeystore { + + Optional<PublicKey> getPublicKey(AthenzService service, String keyId); + + default void preloadKeys(AthenzService service) { /* Default implementation is noop */ } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsClient.java new file mode 100644 index 00000000000..a44f1af0d2a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsClient.java @@ -0,0 +1,15 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; + +import java.util.List; + +/** + * @author bjorncs + */ +public interface ZtsClient { + + List<AthensDomain> getTenantDomainsForUser(AthenzPrincipal principal); + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsException.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsException.java new file mode 100644 index 00000000000..cb0b21ba459 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/ZtsException.java @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz; + +import com.yahoo.athenz.zts.ZTSClientException; + +/** + * @author bjorncs + */ +public class ZtsException extends RuntimeException { + + private final int code; + + public ZtsException(ZTSClientException e) { + super(e.getMessage(), e); + this.code = e.getCode(); + } + + + public int getCode() { + return code; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java new file mode 100644 index 00000000000..de771ff2e17 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilter.java @@ -0,0 +1,71 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.filter; + +import com.google.inject.Inject; +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.FastContentWriter; +import com.yahoo.jdisc.handler.ResponseDispatch; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.jdisc.http.server.jetty.ErrorResponseContentCreator; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; + +import java.util.Optional; +import java.util.concurrent.Executor; + +/** + * Performs authentication by validating the principal token (NToken) header. + * + * @author bjorncs + */ +public class AthenzPrincipalFilter implements SecurityRequestFilter { + + private final ErrorResponseContentCreator responseCreator = new ErrorResponseContentCreator(); + private final NTokenValidator validator; + private final String principalTokenHeader; + + /** + * @param executor to preload the ZMS public keys with + */ + @Inject + public AthenzPrincipalFilter(ZmsKeystore zmsKeystore, Executor executor, AthenzConfig config) { + this(new NTokenValidator(zmsKeystore), executor, config.principalHeaderName()); + } + + AthenzPrincipalFilter(NTokenValidator validator, Executor executor, String principalTokenHeader) { + this.validator = validator; + this.principalTokenHeader = principalTokenHeader; + executor.execute(validator::preloadPublicKeys); + } + + @Override + public void filter(DiscFilterRequest request, ResponseHandler responseHandler) { + String rawToken = request.getHeader(principalTokenHeader); + if (rawToken == null || rawToken.isEmpty()) { + sendUnauthorized(request, responseHandler, "NToken is missing"); + return; + } + try { + AthenzPrincipal principal = validator.validate(new NToken(rawToken)); + request.setUserPrincipal(principal); + request.setRemoteUser(principal.getName()); + } catch (InvalidTokenException e) { + sendUnauthorized(request, responseHandler, e.getMessage()); + } + } + + private void sendUnauthorized(DiscFilterRequest request, ResponseHandler responseHandler, String message) { + try (FastContentWriter writer = ResponseDispatch.newInstance(Response.Status.UNAUTHORIZED) + .connectFastWriter(responseHandler)) { + writer.write( + responseCreator.createErrorContent( + request.getRequestURI(), Response.Status.UNAUTHORIZED, Optional.of(message))); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java new file mode 100644 index 00000000000..f43d2d8e80e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidator.java @@ -0,0 +1,67 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.filter; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; + +import java.security.PublicKey; +import java.util.Optional; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.controller.athenz.AthenzUtils.ZMS_ATHENZ_SERVICE; + +/** + * Validates the content of an NToken: + * 1) Verifies that the token is signed by the sys.auth.zms service (by validating the signature) + * 2) Verifies that the token is not expired + * + * @author bjorncs + */ +class NTokenValidator { + + private static final Logger log = Logger.getLogger(NTokenValidator.class.getName()); + + private final ZmsKeystore keystore; + + NTokenValidator(ZmsKeystore keystore) { + this.keystore = keystore; + } + + void preloadPublicKeys() { + keystore.preloadKeys(ZMS_ATHENZ_SERVICE); + } + + AthenzPrincipal validate(NToken token) throws InvalidTokenException { + PublicKey zmsPublicKey = getPublicKey(token.getKeyId()) + .orElseThrow(() -> new InvalidTokenException("NToken has an unknown keyId")); + validateSignatureAndExpiration(token, zmsPublicKey); + return token.getPrincipal(); + } + + private Optional<PublicKey> getPublicKey(String keyId) throws InvalidTokenException { + try { + return keystore.getPublicKey(ZMS_ATHENZ_SERVICE, keyId); + } catch (Exception e) { + logDebug(e.getMessage()); + throw new InvalidTokenException("Failed to retrieve public key"); + } + } + + private static void validateSignatureAndExpiration(NToken token, PublicKey zmsPublicKey) throws InvalidTokenException { + try { + token.validateSignatureAndExpiration(zmsPublicKey); + } catch (InvalidTokenException e) { + // The underlying error message is not user friendly + logDebug(e.getMessage()); + throw new InvalidTokenException("NToken is expired or has invalid signature"); + } + } + + private static void logDebug(String message) { + log.log(LogLevel.DEBUG, "Failed to validate NToken: " + message); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java new file mode 100644 index 00000000000..292841d89f3 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -0,0 +1,101 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.impl; + +import com.google.inject.Inject; +import com.yahoo.athenz.auth.Principal; +import com.yahoo.athenz.auth.impl.PrincipalAuthority; +import com.yahoo.athenz.auth.impl.SimplePrincipal; +import com.yahoo.athenz.auth.impl.SimpleServiceIdentityProvider; +import com.yahoo.athenz.auth.token.PrincipalToken; +import com.yahoo.athenz.auth.util.Crypto; +import com.yahoo.athenz.zms.ZMSClient; +import com.yahoo.athenz.zts.ZTSClient; +import com.yahoo.vespa.hosted.controller.api.integration.security.KeyService; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; +import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; + +import java.security.PrivateKey; +import java.util.concurrent.TimeUnit; + +/** + * @author bjorncs + */ +public class AthenzClientFactoryImpl implements AthenzClientFactory { + + private final KeyService secretService; + private final AthenzConfig config; + private final AthenzPrincipalAuthority athenzPrincipalAuthority; + + @Inject + public AthenzClientFactoryImpl(KeyService secretService, AthenzConfig config) { + this.secretService = secretService; + this.config = config; + this.athenzPrincipalAuthority = new AthenzPrincipalAuthority(config.principalHeaderName()); + } + + /** + * @return A ZMS client instance with the service identity as principal. + */ + @Override + public ZmsClient createZmsClientWithServicePrincipal() { + return new ZmsClientImpl(new ZMSClient(config.zmsUrl(), createServicePrincipal()), config); + } + + /** + * @return A ZTS client instance with the service identity as principal. + */ + @Override + public ZtsClient createZtsClientWithServicePrincipal() { + return new ZtsClientImpl(new ZTSClient(config.ztsUrl(), createServicePrincipal()), config); + } + + /** + * @return A ZMS client created with a dual principal representing both the tenant admin and the service identity. + */ + @Override + public ZmsClient createZmsClientWithAuthorizedServiceToken(NToken authorizedServiceToken) { + PrincipalToken signedToken = new PrincipalToken(authorizedServiceToken.getToken()); + AthenzConfig.Service service = config.service(); + signedToken.signForAuthorizedService( + config.domain() + "." + service.name(), service.publicKeyId(), getServicePrivateKey()); + + Principal dualPrincipal = SimplePrincipal.create( + "yby", signedToken.getName(), signedToken.getSignedToken(), athenzPrincipalAuthority); + return new ZmsClientImpl(new ZMSClient(config.zmsUrl(), dualPrincipal), config); + + } + + private Principal createServicePrincipal() { + AthenzConfig.Service service = config.service(); + // TODO bjorncs: Cache principal token + SimpleServiceIdentityProvider identityProvider = + new SimpleServiceIdentityProvider( + athenzPrincipalAuthority, config.domain(), service.name(), + getServicePrivateKey(), service.publicKeyId(), /*tokenTimeout*/TimeUnit.HOURS.toSeconds(1)); + return identityProvider.getIdentity(config.domain(), service.name()); + } + + private PrivateKey getServicePrivateKey() { + AthenzConfig.Service service = config.service(); + String privateKey = secretService.getSecret(service.privateKeySecretName(), service.privateKeyVersion()).trim(); + return Crypto.loadPrivateKey(privateKey); + } + + private static class AthenzPrincipalAuthority extends PrincipalAuthority { + private final String principalHeaderName; + + public AthenzPrincipalAuthority(String principalHeaderName) { + this.principalHeaderName = principalHeaderName; + } + + @Override + public String getHeader() { + return principalHeaderName; + } + } + + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java new file mode 100644 index 00000000000..48bd8af7dfe --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsClientImpl.java @@ -0,0 +1,217 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.impl; + +import com.yahoo.athenz.auth.util.Crypto; +import com.yahoo.athenz.zms.DomainList; +import com.yahoo.athenz.zms.ProviderResourceGroupRoles; +import com.yahoo.athenz.zms.PublicKeyEntry; +import com.yahoo.athenz.zms.ServiceIdentity; +import com.yahoo.athenz.zms.Tenancy; +import com.yahoo.athenz.zms.TenantRoleAction; +import com.yahoo.athenz.zms.ZMSClient; +import com.yahoo.athenz.zms.ZMSClientException; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPublicKey; +import com.yahoo.vespa.hosted.controller.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; +import java.util.logging.Logger; + +import static java.util.stream.Collectors.toList; + +/** + * @author bjorncs + */ +public class ZmsClientImpl implements ZmsClient { + + private static final Logger log = Logger.getLogger(ZmsClientImpl.class.getName()); + private final ZMSClient zmsClient; + private final AthenzService service; + + ZmsClientImpl(ZMSClient zmsClient, AthenzConfig config) { + this.zmsClient = zmsClient; + this.service = new AthenzService(config.domain(), config.service().name()); + } + + @Override + public void createTenant(AthensDomain tenantDomain) { + log("putTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); + runOrThrow(() -> { + Tenancy tenancy = new Tenancy() + .setDomain(tenantDomain.id()) + .setService(service.toFullServiceName()) + .setResourceGroups(Collections.emptyList()); + zmsClient.putTenancy(tenantDomain.id(), service.toFullServiceName(), /*auditref*/null, tenancy); + }); + } + + @Override + public void deleteTenant(AthensDomain tenantDomain) { + log("deleteTenancy(tenantDomain=%s, service=%s)", tenantDomain, service); + runOrThrow(() -> zmsClient.deleteTenancy(tenantDomain.id(), service.toFullServiceName(), /*auditref*/null)); + } + + @Override + public void addApplication(AthensDomain tenantDomain, ApplicationId applicationName) { + List<TenantRoleAction> tenantRoleActions = createTenantRoleActions(); + log("putProviderResourceGroupRoles(" + + "tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s, roleActions=%s)", + tenantDomain, service.getDomain().getName(), service.getServiceName(), applicationName, tenantRoleActions); + runOrThrow(() -> { + ProviderResourceGroupRoles resourceGroupRoles = new ProviderResourceGroupRoles() + .setDomain(service.getDomain().getName()) + .setService(service.getServiceName()) + .setTenant(tenantDomain.id()) + .setResourceGroup(applicationName.id()) + .setRoles(tenantRoleActions); + zmsClient.putProviderResourceGroupRoles( + tenantDomain.id(), service.getDomain().getName(), service.getServiceName(), + applicationName.id(), /*auditref*/null, resourceGroupRoles); + }); + } + + @Override + public void deleteApplication(AthensDomain tenantDomain, ApplicationId applicationName) { + log("deleteProviderResourceGroupRoles(tenantDomain=%s, providerDomain=%s, service=%s, resourceGroup=%s)", + tenantDomain, service.getDomain().getName(), service.getServiceName(), applicationName); + runOrThrow(() -> { + zmsClient.deleteProviderResourceGroupRoles( + tenantDomain.id(), service.getDomain().getName(), service.getServiceName(), applicationName.id(), /*auditref*/null); + }); + } + + @Override + public boolean hasApplicationAccess( + AthenzPrincipal principal, ApplicationAction action, AthensDomain tenantDomain, ApplicationId applicationName) { + return hasAccess( + action.name(), applicationResourceString(tenantDomain, applicationName), principal); + } + + @Override + public boolean hasTenantAdminAccess(AthenzPrincipal principal, AthensDomain tenantDomain) { + return hasAccess(TenantAction._modify_.name(), tenantResourceString(tenantDomain), principal); + } + + /** + * Used when creating tenancies. As there are no tenancy policies at this point, + * we cannot use {@link #hasTenantAdminAccess(AthenzPrincipal, AthensDomain)} + */ + @Override + public boolean isDomainAdmin(AthenzPrincipal principal, AthensDomain domain) { + log("getMembership(domain=%s, role=%s, principal=%s)", domain, "admin", principal); + return getOrThrow( + () -> zmsClient.getMembership(domain.id(), "admin", principal.toYRN()).getIsMember()); + } + + @Override + public List<AthensDomain> getDomainList(String prefix) { + log.log(LogLevel.DEBUG, String.format("getDomainList(prefix=%s)", prefix)); + return getOrThrow( + () -> { + DomainList domainList = zmsClient.getDomainList( + /*limit*/null, /*skip*/null, prefix, /*depth*/null, /*domain*/null, + /*productId*/ null, /*modifiedSince*/null); + return toAthensDomains(domainList.getNames()); + }); + } + + @Override + public AthenzPublicKey getPublicKey(AthenzService service, String keyId) { + log("getPublicKeyEntry(domain=%s, service=%s, keyId=%s)", service.getDomain().id(), service.getServiceName(), keyId); + return getOrThrow(() -> { + PublicKeyEntry entry = zmsClient.getPublicKeyEntry(service.getDomain().id(), service.getServiceName(), keyId); + return fromYbase64EncodedKey(entry.getKey(), keyId); + }); + } + + @Override + public List<AthenzPublicKey> getPublicKeys(AthenzService service) { + log("getServiceIdentity(domain=%s, service=%s)", service.getDomain().id(), service.getServiceName()); + return getOrThrow(() -> { + ServiceIdentity serviceIdentity = zmsClient.getServiceIdentity(service.getDomain().id(), service.getServiceName()); + return toAthensPublicKeys(serviceIdentity.getPublicKeys()); + }); + } + + private static AthenzPublicKey fromYbase64EncodedKey(String encodedKey, String keyId) { + return new AthenzPublicKey(Crypto.loadPublicKey(Crypto.ybase64DecodeString(encodedKey)), keyId); + } + + private static List<TenantRoleAction> createTenantRoleActions() { + return Arrays.stream(ApplicationAction.values()) + .map(action -> new TenantRoleAction().setAction(action.name()).setRole(action.roleName)) + .collect(toList()); + } + + private static List<AthensDomain> toAthensDomains(List<String> domains) { + return domains.stream().map(AthensDomain::new).collect(toList()); + } + + private static List<AthenzPublicKey> toAthensPublicKeys(List<PublicKeyEntry> publicKeys) { + return publicKeys.stream() + .map(entry -> fromYbase64EncodedKey(entry.getKey(), entry.getId())) + .collect(toList()); + } + + private boolean hasAccess(String action, String resource, AthenzPrincipal principal) { + log("getAccess(action=%s, resource=%s, principal=%s)", action, resource, principal); + return getOrThrow( + () -> zmsClient.getAccess(action, resource, /*trustDomain*/null, principal.toYRN()).getGranted()); + } + + private static void log(String format, Object... args) { + log.log(LogLevel.DEBUG, String.format(format, args)); + } + + private static void runOrThrow(Runnable wrappedCode) { + try { + wrappedCode.run(); + } catch (ZMSClientException e) { + logWarning(e); + throw new ZmsException(e); + } + } + + private static <T> T getOrThrow(Supplier<T> wrappedCode) { + try { + return wrappedCode.get(); + } catch (ZMSClientException e) { + logWarning(e); + throw new ZmsException(e); + } + } + + private static void logWarning(ZMSClientException e) { + log.warning("Error from Athens: " + e.getMessage()); + } + + private String resourceStringPrefix(AthensDomain tenantDomain) { + return String.format("%s:service.%s.tenant.%s", + service.getDomain().getName(), service.getServiceName(), tenantDomain.id()); + } + + private String tenantResourceString(AthensDomain tenantDomain) { + return resourceStringPrefix(tenantDomain) + ".wildcard"; + } + + private String applicationResourceString(AthensDomain tenantDomain, ApplicationId applicationName) { + return resourceStringPrefix(tenantDomain) + "." + "res_group" + "." + applicationName.id() + ".wildcard"; + } + + private enum TenantAction { + // This is meant to match only the '*' action of the 'admin' role. + // If needed, we can replace it with 'create', 'delete' etc. later. + _modify_ + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java new file mode 100644 index 00000000000..67f22d42f92 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZmsKeystoreImpl.java @@ -0,0 +1,118 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.impl; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPublicKey; +import com.yahoo.vespa.hosted.controller.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.ZmsException; +import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; + +import java.security.PublicKey; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Logger; + +/** + * Downloads and caches public keys for Athens services. + * + * @author bjorncs + */ +public class ZmsKeystoreImpl implements ZmsKeystore { + private static final Logger log = Logger.getLogger(ZmsKeystoreImpl.class.getName()); + + private final Map<FullKeyId, PublicKey> cachedKeys = new ConcurrentHashMap<>(); + private final AthenzClientFactory athenzClientFactory; + + ZmsKeystoreImpl(AthenzClientFactory factory) { + this.athenzClientFactory = factory; + } + + @Override + public Optional<PublicKey> getPublicKey(AthenzService service, String keyId) { + FullKeyId fullKeyId = new FullKeyId(service, keyId); + PublicKey cachedKey = cachedKeys.get(fullKeyId); + if (cachedKey != null) { + return Optional.of(cachedKey); + } + Optional<PublicKey> downloadedKey = downloadPublicKey(fullKeyId); + downloadedKey.ifPresent(key -> { + log.log(LogLevel.INFO, "Adding key " + fullKeyId + " to the cache"); + cachedKeys.put(fullKeyId, key); + }); + return downloadedKey; + } + + @Override + public void preloadKeys(AthenzService service) { + try { + log.log(LogLevel.INFO, "Downloading keys for " + service); + List<AthenzPublicKey> publicKeys = athenzClientFactory.createZmsClientWithServicePrincipal() + .getPublicKeys(service); + for (AthenzPublicKey publicKey : publicKeys) { + FullKeyId fullKeyId = new FullKeyId(service, publicKey.getKeyId()); + log.log(LogLevel.DEBUG, "Adding key " + fullKeyId + " to the cache"); + cachedKeys.put(fullKeyId, publicKey.getPublicKey()); + } + log.log(LogLevel.INFO, "Successfully downloaded keys for " + service); + } catch (ZmsException e) { + log.log(LogLevel.WARNING, "Failed to download keys for " + service + ": " + e.getMessage()); + } + } + + private Optional<PublicKey> downloadPublicKey(FullKeyId fullKeyId) { + try { + log.log(LogLevel.INFO, "Downloading key " + fullKeyId); + AthenzPublicKey publicKey = athenzClientFactory.createZmsClientWithServicePrincipal() + .getPublicKey(fullKeyId.service, fullKeyId.keyId); + return Optional.of(publicKey.getPublicKey()); + } catch (ZmsException e) { + if (e.getCode() == 404) { // Key does not exist + log.log(LogLevel.INFO, "Key " + fullKeyId + " not found"); + return Optional.empty(); + } + String msg = String.format("Unable to retrieve public key from Athens (%s): %s", fullKeyId, e.getMessage()); + throw createException(msg, e); + } + } + + private static RuntimeException createException(String message, Exception cause) { + log.log(LogLevel.ERROR, message); + return new RuntimeException(message, cause); + } + + private static class FullKeyId { + private final AthenzService service; + private final String keyId; + + private FullKeyId(AthenzService service, String keyId) { + this.service = service; + this.keyId = keyId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + FullKeyId fullKeyId1 = (FullKeyId) o; + return Objects.equals(service, fullKeyId1.service) && + Objects.equals(keyId, fullKeyId1.keyId); + } + + @Override + public int hashCode() { + return Objects.hash(service, keyId); + } + + @Override + public String toString() { + return "FullKeyId{" + + "service=" + service + + ", keyId='" + keyId + '\'' + + '}'; + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java new file mode 100644 index 00000000000..d964cc967e8 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/ZtsClientImpl.java @@ -0,0 +1,51 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.impl; + +import com.yahoo.athenz.zts.TenantDomains; +import com.yahoo.athenz.zts.ZTSClient; +import com.yahoo.athenz.zts.ZTSClientException; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; +import com.yahoo.vespa.hosted.controller.athenz.ZtsException; +import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; + +import java.util.List; +import java.util.logging.Logger; + +import static java.util.stream.Collectors.toList; + +/** + * @author bjorncs + */ +public class ZtsClientImpl implements ZtsClient { + + private static final Logger log = Logger.getLogger(ZtsClientImpl.class.getName()); + + private final ZTSClient ztsClient; + private final AthenzService service; + + public ZtsClientImpl(ZTSClient ztsClient, AthenzConfig config) { + this.ztsClient = ztsClient; + this.service = new AthenzService(config.domain(), config.service().name()); + } + + @Override + public List<AthensDomain> getTenantDomainsForUser(AthenzPrincipal principal) { + log.log(LogLevel.DEBUG, String.format( + "getTenantDomains(domain=%s, username=%s, rolename=admin, service=%s)", + service.getDomain().getName(), principal, service.getServiceName())); + try { + TenantDomains domains = ztsClient.getTenantDomains( + service.getDomain().getName(), principal.toYRN(), "admin", service.getServiceName()); + return domains.getTenantDomainNames().stream() + .map(AthensDomain::new) + .collect(toList()); + } catch (ZTSClientException e) { + throw new ZtsException(e); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthensDbMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthensDbMock.java new file mode 100644 index 00000000000..55fe435c9be --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthensDbMock.java @@ -0,0 +1,73 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.mock; + +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * @author bjorncs + */ +public class AthensDbMock { + + public final Map<AthensDomain, Domain> domains = new HashMap<>(); + + public AthensDbMock addDomain(Domain domain) { + domains.put(domain.name, domain); + return this; + } + + public static class Domain { + + public final AthensDomain name; + public final Set<AthenzPrincipal> admins = new HashSet<>(); + public final Set<AthenzPrincipal> tenantAdmins = new HashSet<>(); + public final Map<ApplicationId, Application> applications = new HashMap<>(); + public boolean isVespaTenant = false; + + public Domain(AthensDomain name) { + this.name = name; + } + + public Domain admin(AthenzPrincipal user) { + admins.add(user); + return this; + } + + public Domain tenantAdmin(AthenzPrincipal user) { + tenantAdmins.add(user); + return this; + } + + /** + * Simulates establishing Vespa tenancy in Athens. + */ + public void markAsVespaTenant() { + isVespaTenant = true; + } + + } + + public static class Application { + + public final Map<ApplicationAction, Set<AthenzPrincipal>> acl = new HashMap<>(); + + public Application() { + acl.put(ApplicationAction.deploy, new HashSet<>()); + acl.put(ApplicationAction.read, new HashSet<>()); + acl.put(ApplicationAction.write, new HashSet<>()); + } + + public Application addRoleMember(ApplicationAction action, AthenzPrincipal user) { + acl.get(action).add(user); + return this; + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java new file mode 100644 index 00000000000..92c7ba5a007 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/AthenzClientFactoryMock.java @@ -0,0 +1,56 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.mock; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; + +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +public class AthenzClientFactoryMock extends AbstractComponent implements AthenzClientFactory { + + private static final Logger log = Logger.getLogger(AthenzClientFactoryMock.class.getName()); + + private final AthensDbMock athens; + + public AthenzClientFactoryMock() { + this(new AthensDbMock()); + } + + public AthenzClientFactoryMock(AthensDbMock athens) { + this.athens = athens; + } + + public AthensDbMock getSetup() { + return athens; + } + + @Override + public ZmsClient createZmsClientWithServicePrincipal() { + log("createZmsClientWithServicePrincipal()"); + return new ZmsClientMock(athens); + } + + @Override + public ZtsClient createZtsClientWithServicePrincipal() { + log("createZtsClientWithServicePrincipal()"); + return new ZtsClientMock(athens); + } + + @Override + public ZmsClient createZmsClientWithAuthorizedServiceToken(NToken authorizedServiceToken) { + log("createZmsClientWithAuthorizedServiceToken(authorizedServiceToken='%s')", authorizedServiceToken); + return new ZmsClientMock(athens); + } + + private static void log(String format, Object... args) { + log.log(Level.INFO, String.format(format, args)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java new file mode 100644 index 00000000000..bba7d410bf7 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -0,0 +1,121 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.mock; + +import com.yahoo.athenz.zms.ZMSClientException; +import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPublicKey; +import com.yahoo.vespa.hosted.controller.athenz.AthenzService; +import com.yahoo.vespa.hosted.controller.athenz.ZmsClient; +import com.yahoo.vespa.hosted.controller.athenz.ZmsException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +public class ZmsClientMock implements ZmsClient { + + private static final Logger log = Logger.getLogger(ZmsClientMock.class.getName()); + + private final AthensDbMock athens; + + public ZmsClientMock(AthensDbMock athens) { + this.athens = athens; + } + + @Override + public void createTenant(AthensDomain tenantDomain) { + log("createTenant(tenantDomain='%s')", tenantDomain); + getDomainOrThrow(tenantDomain, false).isVespaTenant = true; + } + + @Override + public void deleteTenant(AthensDomain tenantDomain) { + log("deleteTenant(tenantDomain='%s')", tenantDomain); + AthensDbMock.Domain domain = getDomainOrThrow(tenantDomain, false); + domain.isVespaTenant = false; + domain.applications.clear(); + domain.tenantAdmins.clear(); + } + + @Override + public void addApplication(AthensDomain tenantDomain, ApplicationId applicationName) { + log("addApplication(tenantDomain='%s', applicationName='%s')", tenantDomain, applicationName); + AthensDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); + if (!domain.applications.containsKey(applicationName)) { + domain.applications.put(applicationName, new AthensDbMock.Application()); + } + } + + @Override + public void deleteApplication(AthensDomain tenantDomain, ApplicationId applicationName) { + log("addApplication(tenantDomain='%s', applicationName='%s')", tenantDomain, applicationName); + getDomainOrThrow(tenantDomain, true).applications.remove(applicationName); + } + + @Override + public boolean hasApplicationAccess(AthenzPrincipal principal, ApplicationAction action, AthensDomain tenantDomain, ApplicationId applicationName) { + log("hasApplicationAccess(principal='%s', action='%s', tenantDomain='%s', applicationName='%s')", + principal, action, tenantDomain, applicationName); + AthensDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); + AthensDbMock.Application application = domain.applications.get(applicationName); + if (application == null) { + throw zmsException(400, "Application '%s' not found", applicationName); + } + return domain.admins.contains(principal) || application.acl.get(action).contains(principal); + } + + @Override + public boolean hasTenantAdminAccess(AthenzPrincipal principal, AthensDomain tenantDomain) { + log("hasTenantAdminAccess(principal='%s', tenantDomain='%s')", principal, tenantDomain); + return isDomainAdmin(principal, tenantDomain) || + getDomainOrThrow(tenantDomain, true).tenantAdmins.contains(principal); + } + + @Override + public boolean isDomainAdmin(AthenzPrincipal principal, AthensDomain domain) { + log("isDomainAdmin(principal='%s', domain='%s')", principal, domain); + return getDomainOrThrow(domain, false).admins.contains(principal); + } + + @Override + public List<AthensDomain> getDomainList(String prefix) { + log("getDomainList()"); + return new ArrayList<>(athens.domains.keySet()); + } + + @Override + public AthenzPublicKey getPublicKey(AthenzService service, String keyId) { + throw new UnsupportedOperationException(); + } + + @Override + public List<AthenzPublicKey> getPublicKeys(AthenzService service) { + throw new UnsupportedOperationException(); + } + + private AthensDbMock.Domain getDomainOrThrow(AthensDomain domainName, boolean verifyVespaTenant) { + AthensDbMock.Domain domain = Optional.ofNullable(athens.domains.get(domainName)) + .orElseThrow(() -> zmsException(400, "Domain '%s' not found", domainName)); + if (verifyVespaTenant && !domain.isVespaTenant) { + throw zmsException(400, "Domain not a Vespa tenant: '%s'", domainName); + } + return domain; + } + + private static ZmsException zmsException(int code, String message, Object... args) { + return new ZmsException(new ZMSClientException(code, String.format(message, args))); + } + + private static void log(String format, Object... args) { + log.log(Level.INFO, String.format(format, args)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java new file mode 100644 index 00000000000..fa41dcc6446 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java @@ -0,0 +1,34 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.mock; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.ZtsClient; + +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static java.util.stream.Collectors.toList; + +/** + * @author bjorncs + */ +public class ZtsClientMock implements ZtsClient { + private static final Logger log = Logger.getLogger(ZtsClientMock.class.getName()); + + private final AthensDbMock athens; + + public ZtsClientMock(AthensDbMock athens) { + this.athens = athens; + } + + @Override + public List<AthensDomain> getTenantDomainsForUser(AthenzPrincipal principal) { + log.log(Level.INFO, "getTenantDomainsForUser(principal='%s')", principal); + return athens.domains.values().stream() + .filter(domain -> domain.tenantAdmins.contains(principal) || domain.admins.contains(principal)) + .map(domain -> domain.name) + .collect(toList()); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index 8f5db8832fa..c807a7f0586 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.Zone; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -15,6 +17,7 @@ import java.time.Duration; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -38,7 +41,7 @@ public class ClusterInfoMaintainer extends Maintainer { return node.membership.clusterId; } - private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeList nodes) { + private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeList nodes, Zone zone) { Map<ClusterSpec.Id, ClusterInfo> infoMap = new HashMap<>(); // Group nodes by clusterid @@ -53,9 +56,24 @@ public class ClusterInfoMaintainer extends Maintainer { //Assume they are all equal and use first node as a representatitve for the cluster NodeList.Node node = clusterNodes.get(0); + // Extract flavor info + double cpu = 0; + double mem = 0; + double disk = 0; + if (zone.nodeFlavors().isPresent()) { + Optional<Flavor> flavorOptional = zone.nodeFlavors().get().getFlavor(node.flavor); + if ((flavorOptional.isPresent())) { + Flavor flavor = flavorOptional.get(); + cpu = flavor.getMinCpuCores(); + mem = flavor.getMinMainMemoryAvailableGb(); + disk = flavor.getMinMainMemoryAvailableGb(); + } + } + // Add to map List<String> hostnames = clusterNodes.stream().map(node1 -> node1.hostname).collect(Collectors.toList()); - ClusterInfo inf = new ClusterInfo(node.flavor, node.cost, ClusterSpec.Type.from(node.membership.clusterType), hostnames); + ClusterInfo inf = new ClusterInfo(node.flavor, node.cost, cpu, mem, disk, + ClusterSpec.Type.from(node.membership.clusterType), hostnames); infoMap.put(new ClusterSpec.Id(id), inf); } @@ -71,7 +89,7 @@ public class ClusterInfoMaintainer extends Maintainer { DeploymentId deploymentId = new DeploymentId(application.id(), deployment.zone()); try { NodeList nodes = controller().applications().configserverClient().getNodeList(deploymentId); - Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes); + Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes, deployment.zone()); Application app = application.with(deployment.withClusterInfo(clusterInfo)); controller.applications().store(app, lock); } catch (IOException ioe) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 880abaaa6f9..fd2e7496ec0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -35,6 +35,7 @@ public class ControllerMaintenance extends AbstractComponent { private final BlockedChangeDeployer blockedChangeDeployer; private final ClusterInfoMaintainer clusterInfoMaintainer; private final ClusterUtilizationMaintainer clusterUtilizationMaintainer; + private final DeploymentMetricsMaintainer deploymentMetricsMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator, @@ -54,6 +55,7 @@ public class ControllerMaintenance extends AbstractComponent { blockedChangeDeployer = new BlockedChangeDeployer(controller, maintenanceInterval, jobControl); clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl); clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); + deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(10), jobControl); } public Upgrader upgrader() { return upgrader; } @@ -74,6 +76,7 @@ public class ControllerMaintenance extends AbstractComponent { blockedChangeDeployer.deconstruct(); clusterUtilizationMaintainer.deconstruct(); clusterInfoMaintainer.deconstruct(); + deploymentMetricsMaintainer.deconstruct(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java new file mode 100644 index 00000000000..d9ef451ffb7 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -0,0 +1,43 @@ +package com.yahoo.vespa.hosted.controller.maintenance;// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; + +import java.time.Duration; + +/** + * Retrieve deployment metrics like qps and document count from the metric service and + * update the applications with this info. + * + * @author smorgrav + */ +public class DeploymentMetricsMaintainer extends Maintainer { + + DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) { + super(controller, duration, jobControl); + } + + @Override + protected void maintain() { + + for (Application application : controller().applications().asList()) { + try (Lock lock = controller().applications().lock(application.id())) { + for (Deployment deployment : application.deployments().values()) { + + MetricsService.DeploymentMetrics metrics = controller().metricsService() + .getDeploymentMetrics(application.id(), deployment.zone()); + + DeploymentMetrics appMetrics = new DeploymentMetrics(metrics.queriesPerSecond(), metrics.writesPerSecond(), + metrics.documentCount(), metrics.queryLatencyMillis(), metrics.writeLatencyMillis()); + + Application app = application.with(deployment.withMetrics(appMetrics)); + controller().applications().store(app, lock); + } + } + } + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 859e322b227..41d1a2d624c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; @@ -85,6 +86,9 @@ public class ApplicationSerializer { private final String clusterInfoField = "clusterInfo"; private final String clusterInfoFlavorField = "flavor"; private final String clusterInfoCostField = "cost"; + private final String clusterInfoCpuField = "flavorCpu"; + private final String clusterInfoMemField = "flavorMem"; + private final String clusterInfoDiskField = "flavorDisk"; private final String clusterInfoTypeField = "clusterType"; private final String clusterInfoHostnamesField = "hostnames"; @@ -95,6 +99,14 @@ public class ApplicationSerializer { private final String clusterUtilsDiskField = "disk"; private final String clusterUtilsDiskBusyField = "diskbusy"; + // Deployment metrics fields + private final String deploymentMetricsField = "metrics"; + private final String deploymentMetricsQPSField = "queriesPerSecond"; + private final String deploymentMetricsWPSField = "writesPerSecond"; + private final String deploymentMetricsDocsField = "documentCount"; + private final String deploymentMetricsQueryLatencyField = "queryLatencyMillis"; + private final String deploymentMetricsWriteLatencyField = "writeLatencyMillis"; + // ------------------ Serialization @@ -123,6 +135,16 @@ public class ApplicationSerializer { toSlime(deployment.revision(), object.setObject(applicationPackageRevisionField)); clusterInfoToSlime(deployment.clusterInfo(), object); clusterUtilsToSlime(deployment.clusterUtils(), object); + metricsToSlime(deployment.metrics(), object); + } + + private void metricsToSlime(DeploymentMetrics metrics, Cursor object) { + Cursor root = object.setObject(deploymentMetricsField); + root.setDouble(deploymentMetricsQPSField, metrics.queriesPerSecond()); + root.setDouble(deploymentMetricsWPSField, metrics.writesPerSecond()); + root.setDouble(deploymentMetricsDocsField, metrics.documentCount()); + root.setDouble(deploymentMetricsQueryLatencyField, metrics.queryLatencyMillis()); + root.setDouble(deploymentMetricsWriteLatencyField, metrics.writeLatencyMillis()); } private void clusterInfoToSlime(Map<ClusterSpec.Id, ClusterInfo> clusters, Cursor object) { @@ -134,7 +156,10 @@ public class ApplicationSerializer { private void toSlime(ClusterInfo info, Cursor object) { object.setString(clusterInfoFlavorField, info.getFlavor()); - object.setLong(clusterInfoCostField, info.getCost()); + object.setLong(clusterInfoCostField, info.getFlavorCost()); + object.setDouble(clusterInfoCpuField, info.getFlavorCPU()); + object.setDouble(clusterInfoMemField, info.getFlavorMem()); + object.setDouble(clusterInfoDiskField, info.getFlavorDisk()); object.setString(clusterInfoTypeField, info.getClusterType().name()); Cursor array = object.setArray(clusterInfoHostnamesField); for (String host : info.getHostnames()) { @@ -223,7 +248,7 @@ public class ApplicationSerializer { Inspector root = slime.get(); ApplicationId id = ApplicationId.fromSerializedForm(root.field(idField).asString()); - DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString()); + DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(root.field(deploymentSpecField).asString(), false); ValidationOverrides validationOverrides = ValidationOverrides.fromXml(root.field(validationOverridesField).asString()); List<Deployment> deployments = deploymentsFromSlime(root.field(deploymentsField)); DeploymentJobs deploymentJobs = deploymentJobsFromSlime(root.field(deploymentJobsField)); @@ -246,7 +271,20 @@ public class ApplicationSerializer { Version.fromString(deploymentObject.field(versionField).asString()), Instant.ofEpochMilli(deploymentObject.field(deployTimeField).asLong()), clusterUtilsMapFromSlime(deploymentObject.field(clusterUtilsField)), - clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField))); + clusterInfoMapFromSlime(deploymentObject.field(clusterInfoField)), + deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField))); + } + + private DeploymentMetrics deploymentMetricsFromSlime(Inspector object) { + + double queriesPerSecond = object.field(deploymentMetricsQPSField).asDouble(); + double writesPerSecond = object.field(deploymentMetricsWPSField).asDouble(); + double documentCount = object.field(deploymentMetricsDocsField).asDouble(); + double queryLatencyMillis = object.field(deploymentMetricsQueryLatencyField).asDouble(); + double writeLatencyMills = object.field(deploymentMetricsWriteLatencyField).asDouble(); + + return new DeploymentMetrics(queriesPerSecond, writesPerSecond, + documentCount, queryLatencyMillis, writeLatencyMills); } private Map<ClusterSpec.Id, ClusterInfo> clusterInfoMapFromSlime(Inspector object) { @@ -274,10 +312,13 @@ public class ApplicationSerializer { String flavor = inspector.field(clusterInfoFlavorField).asString(); int cost = (int)inspector.field(clusterInfoCostField).asLong(); String type = inspector.field(clusterInfoTypeField).asString(); + double flavorCpu = inspector.field(clusterInfoCpuField).asDouble(); + double flavorMem = inspector.field(clusterInfoMemField).asDouble(); + double flavorDisk = inspector.field(clusterInfoDiskField).asDouble(); List<String> hostnames = new ArrayList<>(); inspector.field(clusterInfoHostnamesField).traverse((ArrayTraverser)(int index, Inspector value) -> hostnames.add(value.asString())); - return new ClusterInfo(flavor, cost, ClusterSpec.Type.from(type), hostnames); + return new ClusterInfo(flavor, cost, flavorCpu, flavorMem, flavorDisk, ClusterSpec.Type.from(type), hostnames); } private Zone zoneFromSlime(Inspector object) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index a70e31d9de8..2ca662ccb66 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -4,20 +4,21 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.inject.Inject; import com.yahoo.cloud.config.ClusterInfoConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.zookeeper.ZooKeeperServer; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayDeque; import java.util.Collections; @@ -148,18 +149,6 @@ public class CuratorDb { // -------------- Read and write -------------------------------------------------- - public Version readSystemVersion() { - Optional<byte[]> data = curator.getData(systemVersionPath()); - if (! data.isPresent() || data.get().length == 0) return Vtag.currentVersion; - return Version.fromString(new String(data.get(), StandardCharsets.UTF_8)); - } - - public void writeSystemVersion(Version version) { - NestedTransaction transaction = new NestedTransaction(); - curator.set(systemVersionPath(), version.toString().getBytes(StandardCharsets.UTF_8)); - transaction.commit(); - } - public Set<String> readInactiveJobs() { try { Optional<byte[]> data = curator.getData(inactiveJobsPath()); @@ -199,7 +188,7 @@ public class CuratorDb { } public double readUpgradesPerMinute() { - Optional<byte[]> n = curator.getData(upgradePerMinutePath()); + Optional<byte[]> n = curator.getData(upgradesPerMinutePath()); if (!n.isPresent() || n.get().length == 0) { return 0.5; // Default if value has never been written } @@ -211,10 +200,34 @@ public class CuratorDb { throw new IllegalArgumentException("Upgrades per minute must be >= 0"); } NestedTransaction transaction = new NestedTransaction(); - curator.set(upgradePerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array()); + curator.set(upgradesPerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array()); + transaction.commit(); + } + + public void writeVersionStatus(VersionStatus status) { + VersionStatusSerializer serializer = new VersionStatusSerializer(); + NestedTransaction transaction = new NestedTransaction(); + try { + // TODO: Removes unused data. Remove after October 2017 + if (curator.getData(systemVersionPath()).isPresent()) { + curator.delete(systemVersionPath()); + } + curator.set(versionStatusPath(), SlimeUtils.toJsonBytes(serializer.toSlime(status))); + } catch (IOException e) { + throw new UncheckedIOException("Failed to serialize version status", e); + } transaction.commit(); } + public VersionStatus readVersionStatus() { + Optional<byte[]> data = curator.getData(versionStatusPath()); + if (!data.isPresent() || data.get().length == 0) { + return VersionStatus.empty(); // Default if status has never been written + } + VersionStatusSerializer serializer = new VersionStatusSerializer(); + return serializer.fromSlime(SlimeUtils.jsonToSlime(data.get())); + } + public Optional<byte[]> readProvisionState(String provisionId) { return curator.getData(provisionStatePath().append(provisionId)); } @@ -264,10 +277,12 @@ public class CuratorDb { return root.append("jobQueues").append(jobType.name()); } - private Path upgradePerMinutePath() { + private Path upgradesPerMinutePath() { return root.append("upgrader").append("upgradesPerMinute"); } + private Path versionStatusPath() { return root.append("versionStatus"); } + private Path provisionStatePath() { return root.append("provisioning").append("states"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java new file mode 100644 index 00000000000..e0bc592c82c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -0,0 +1,118 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Serializes VersionStatus to and from slime + * + * @author mpolden + */ +public class VersionStatusSerializer { + + // VersionStatus fields + private static final String versionsField = "versions"; + + // VespaVersion fields + private static final String releaseCommitField = "releaseCommit"; + private static final String releasedAtField = "releasedAt"; + private static final String isCurrentSystemVersionField = "isCurrentSystemVersion"; + private static final String deploymentStatisticsField = "deploymentStatistics"; + private static final String confidenceField = "confidence"; + private static final String configServersField = "configServerHostnames"; + + // DeploymentStatistics fields + private static final String versionField = "version"; + private static final String failingField = "failing"; + private static final String productionField = "production"; + + public Slime toSlime(VersionStatus status) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + versionsToSlime(status.versions(), root.setArray(versionsField)); + return slime; + } + + public VersionStatus fromSlime(Slime slime) { + Inspector root = slime.get(); + return new VersionStatus(vespaVersionsFromSlime(root.field(versionsField))); + } + + private void versionsToSlime(List<VespaVersion> versions, Cursor array) { + versions.forEach(version -> vespaVersionToSlime(version, array.addObject())); + } + + private void vespaVersionToSlime(VespaVersion version, Cursor object) { + object.setString(releaseCommitField, version.releaseCommit()); + object.setLong(releasedAtField, version.releasedAt().toEpochMilli()); + object.setBool(isCurrentSystemVersionField, version.isCurrentSystemVersion()); + deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField)); + object.setString(confidenceField, version.confidence().name()); + configServersToSlime(version.configServerHostnames(), object.setArray(configServersField)); + } + + private void configServersToSlime(Set<String> configServerHostnames, Cursor array) { + configServerHostnames.forEach(array::addString); + } + + private void deploymentStatisticsToSlime(DeploymentStatistics statistics, Cursor object) { + object.setString(versionField, statistics.version().toString()); + applicationsToSlime(statistics.failing(), object.setArray(failingField)); + applicationsToSlime(statistics.production(), object.setArray(productionField)); + } + + private void applicationsToSlime(List<ApplicationId> applications, Cursor array) { + applications.forEach(application -> array.addString(application.serializedForm())); + } + + private List<VespaVersion> vespaVersionsFromSlime(Inspector array) { + List<VespaVersion> versions = new ArrayList<>(); + array.traverse((ArrayTraverser) (i, object) -> versions.add(vespaVersionFromSlime(object))); + return Collections.unmodifiableList(versions); + } + + private VespaVersion vespaVersionFromSlime(Inspector object) { + return new VespaVersion(deploymentStatisticsFromSlime(object.field(deploymentStatisticsField)), + object.field(releaseCommitField).asString(), + Instant.ofEpochMilli(object.field(releasedAtField).asLong()), + object.field(isCurrentSystemVersionField).asBool(), + configServersFromSlime(object.field(configServersField)), + VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) + ); + } + + private Set<String> configServersFromSlime(Inspector array) { + Set<String> configServerHostnames = new LinkedHashSet<>(); + array.traverse((ArrayTraverser) (i, entry) -> configServerHostnames.add(entry.asString())); + return Collections.unmodifiableSet(configServerHostnames); + } + + private DeploymentStatistics deploymentStatisticsFromSlime(Inspector object) { + return new DeploymentStatistics(Version.fromString(object.field(versionField).asString()), + applicationsFromSlime(object.field(failingField)), + applicationsFromSlime(object.field(productionField))); + } + + private List<ApplicationId> applicationsFromSlime(Inspector array) { + List<ApplicationId> applications = new ArrayList<>(); + array.traverse((ArrayTraverser) (i, entry) -> applications.add( + ApplicationId.fromSerializedForm(entry.asString())) + ); + return Collections.unmodifiableList(applications); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 42e89e7893f..cb6c47ad299 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.google.common.base.Joiner; +import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -50,9 +51,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; -import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; -import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus; @@ -62,10 +60,15 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; -import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; +import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.Path; @@ -105,11 +108,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private final Controller controller; private final Authorizer authorizer; + private final AthenzClientFactory athenzClientFactory; - public ApplicationApiHandler(Executor executor, AccessLog accessLog, Controller controller, Authorizer authorizer) { + @Inject + public ApplicationApiHandler(Executor executor, AccessLog accessLog, Controller controller, Authorizer authorizer, + AthenzClientFactory athenzClientFactory) { super(executor, accessLog); this.controller = controller; this.authorizer = authorizer; + this.athenzClientFactory = athenzClientFactory; } @Override @@ -433,20 +440,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { toSlime(appCost, costObject); // Metrics - com.yahoo.config.provision.ApplicationId applicationId = com.yahoo.config.provision.ApplicationId.from(tenantName, applicationName, instanceName); - Zone zoneId = new Zone(Environment.from(environment), RegionName.from(region)); - try { - MetricsService.DeploymentMetrics metrics = controller.metricsService().getDeploymentMetrics(applicationId, zoneId); - Cursor metricsObject = response.setObject("metrics"); - metricsObject.setDouble("queriesPerSecond", metrics.queriesPerSecond()); - metricsObject.setDouble("writesPerSecond", metrics.writesPerSecond()); - metricsObject.setDouble("documentCount", metrics.documentCount()); - metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis()); - metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis()); - } - catch (RuntimeException e) { - log.log(Level.WARNING, "Failed getting Yamas metrics", Exceptions.toMessageString(e)); - } + DeploymentMetrics metrics = deployment.metrics(); + Cursor metricsObject = response.setObject("metrics"); + metricsObject.setDouble("queriesPerSecond", metrics.queriesPerSecond()); + metricsObject.setDouble("writesPerSecond", metrics.writesPerSecond()); + metricsObject.setDouble("documentCount", metrics.documentCount()); + metricsObject.setDouble("queryLatencyMillis", metrics.queryLatencyMillis()); + metricsObject.setDouble("writeLatencyMillis", metrics.writeLatencyMillis()); return new SlimeJsonResponse(slime); } @@ -760,10 +760,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); - DeployAuthorizer deployAuthorizer = new DeployAuthorizer(controller.athens(), controller.zoneRegistry()); + DeployAuthorizer deployAuthorizer = new DeployAuthorizer(controller.zoneRegistry(), athenzClientFactory); Tenant tenant = controller.tenants().tenant(new TenantId(tenantName)).orElseThrow(() -> new NotExistsException(new TenantId(tenantName))); Principal principal = authorizer.getPrincipal(request); - if (principal instanceof AthensPrincipal) { + if (principal instanceof AthenzPrincipal) { deployAuthorizer.throwIfUnauthorizedForDeploy(principal, Environment.from(environment), tenant, @@ -1084,8 +1084,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { public static void toSlime(DeploymentCost deploymentCost, Cursor object) { object.setLong("tco", (long)deploymentCost.getTco()); + object.setLong("waste", (long)deploymentCost.getWaste()); object.setDouble("utilization", deploymentCost.getUtilization()); - object.setDouble("waste", deploymentCost.getWaste()); Cursor clustersObject = object.setObject("cluster"); for (Map.Entry<String, ClusterCost> clusterEntry : deploymentCost.getCluster().entrySet()) toSlime(clusterEntry.getValue(), clustersObject.setObject(clusterEntry.getKey())); @@ -1096,8 +1096,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { object.setString("resource", getResourceName(clusterCost.getResultUtilization())); object.setDouble("utilization", clusterCost.getResultUtilization().getMaxUtilization()); object.setLong("tco", (int)clusterCost.getTco()); - object.setString("flavor", clusterCost.getClusterInfo().getFlavor()); object.setLong("waste", (int)clusterCost.getWaste()); + object.setString("flavor", clusterCost.getClusterInfo().getFlavor()); + object.setDouble("flavorCost", clusterCost.getClusterInfo().getFlavorCost()); + object.setDouble("flavorCpu", clusterCost.getClusterInfo().getFlavorCPU()); + object.setDouble("flavorMem", clusterCost.getClusterInfo().getFlavorMem()); + object.setDouble("flavorDisk", clusterCost.getClusterInfo().getFlavorDisk()); object.setString("type", clusterCost.getClusterInfo().getClusterType().name()); Cursor utilObject = object.setObject("util"); utilObject.setDouble("cpu", clusterCost.getResultUtilization().getCpu()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java index 8dff39779b9..84e731ec994 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/Authorizer.java @@ -11,10 +11,10 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsClientFactory; -import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; -import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; -import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.athenz.NToken; import com.yahoo.vespa.hosted.controller.common.ContextAttributes; import com.yahoo.vespa.hosted.controller.restapi.filter.NTokenRequestFilter; import com.yahoo.vespa.hosted.controller.restapi.filter.UnauthenticatedUserPrincipal; @@ -47,15 +47,13 @@ public class Authorizer { new UserId("screwdriver-test")); private final Controller controller; - private final ZmsClientFactory zmsClientFactory; + private final AthenzClientFactory athenzClientFactory; private final EntityService entityService; - private final Athens athens; - public Authorizer(Controller controller, EntityService entityService) { + public Authorizer(Controller controller, EntityService entityService, AthenzClientFactory athenzClientFactory) { this.controller = controller; - this.zmsClientFactory = controller.athens().zmsClientFactory(); + this.athenzClientFactory = athenzClientFactory; this.entityService = entityService; - this.athens = controller.athens(); } public void throwIfUnauthorized(TenantId tenantId, HttpRequest request) throws ForbiddenException { @@ -90,19 +88,19 @@ public class Authorizer { public Optional<NToken> getNToken(HttpRequest request) { String nTokenHeader = (String)request.getJDiscRequest().context().get(NTokenRequestFilter.NTOKEN_HEADER); - return Optional.ofNullable(nTokenHeader).map(athens::nTokenFrom); + return Optional.ofNullable(nTokenHeader).map(NToken::new); } public boolean isSuperUser(HttpRequest request) { // TODO Check membership of admin role in Vespa's Athens domain - return isMemberOfVespaBouncerGroup(request) || isScrewdriverPrincipal(athens, getPrincipal(request)); + return isMemberOfVespaBouncerGroup(request) || isScrewdriverPrincipal(getPrincipal(request)); } - public static boolean isScrewdriverPrincipal(Athens athens, Principal principal) { + public static boolean isScrewdriverPrincipal(Principal principal) { if (principal instanceof UnauthenticatedUserPrincipal) // Host-based authentication return SCREWDRIVER_USERS.contains(new UserId(principal.getName())); - else if (principal instanceof AthensPrincipal) - return ((AthensPrincipal)principal).getDomain().equals(athens.screwdriverDomain()); + else if (principal instanceof AthenzPrincipal) + return ((AthenzPrincipal)principal).getDomain().equals(AthenzUtils.SCREWDRIVER_DOMAIN); else return false; } @@ -126,13 +124,13 @@ public class Authorizer { } private boolean isAthensTenantAdmin(UserId userId, AthensDomain tenantDomain) { - return zmsClientFactory.createClientWithServicePrincipal() - .hasTenantAdminAccess(athens.principalFrom(userId), tenantDomain); + return athenzClientFactory.createZmsClientWithServicePrincipal() + .hasTenantAdminAccess(AthenzUtils.createPrincipal(userId), tenantDomain); } public boolean isAthensDomainAdmin(UserId userId, AthensDomain tenantDomain) { - return zmsClientFactory.createClientWithServicePrincipal() - .isDomainAdmin(athens.principalFrom(userId), tenantDomain); + return athenzClientFactory.createZmsClientWithServicePrincipal() + .isDomainAdmin(AthenzUtils.createPrincipal(userId), tenantDomain); } public boolean isGroupMember(UserId userId, UserGroup userGroup) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java index 5c7cdfdae0a..fa82c9239df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/DeployAuthorizer.java @@ -7,11 +7,12 @@ import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ApplicationAction; -import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; -import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; -import com.yahoo.vespa.hosted.controller.api.integration.athens.ZmsException; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.athenz.ZmsException; import com.yahoo.vespa.hosted.controller.restapi.filter.UnauthenticatedUserPrincipal; import javax.ws.rs.ForbiddenException; @@ -30,12 +31,12 @@ public class DeployAuthorizer { private static final Logger log = Logger.getLogger(DeployAuthorizer.class.getName()); - private final Athens athens; private final ZoneRegistry zoneRegistry; + private final AthenzClientFactory athenzClientFactory; - public DeployAuthorizer(Athens athens, ZoneRegistry zoneRegistry) { - this.athens = athens; + public DeployAuthorizer(ZoneRegistry zoneRegistry, AthenzClientFactory athenzClientFactory) { this.zoneRegistry = zoneRegistry; + this.athenzClientFactory = athenzClientFactory; } public void throwIfUnauthorizedForDeploy(Principal principal, @@ -50,7 +51,7 @@ public class DeployAuthorizer { private boolean athensCredentialsRequired(Environment environment, Tenant tenant, ApplicationId applicationId, Principal principal) { if (!environmentRequiresAuthorization(environment)) return false; - if (! isScrewdriverPrincipal(athens, principal)) + if (! isScrewdriverPrincipal(principal)) throw loggedForbiddenException( "Principal '%s' is not a screwdriver principal, and does not have deploy access to application '%s'", principal.getName(), applicationId.toShortString()); @@ -62,10 +63,10 @@ public class DeployAuthorizer { // TODO: inline when deployment via ssh is removed private void checkAthensCredentials(Principal principal, Tenant tenant, ApplicationId applicationId) { AthensDomain domain = tenant.getAthensDomain().get(); - if (! (principal instanceof AthensPrincipal)) + if (! (principal instanceof AthenzPrincipal)) throw loggedForbiddenException("Principal '%s' is not authenticated.", principal.getName()); - AthensPrincipal athensPrincipal = (AthensPrincipal)principal; + AthenzPrincipal athensPrincipal = (AthenzPrincipal)principal; if ( ! hasDeployAccessToAthensApplication(athensPrincipal, domain, applicationId)) throw loggedForbiddenException( "Screwdriver principal '%1$s' does not have deploy access to '%2$s'. " + @@ -95,14 +96,14 @@ public class DeployAuthorizer { if (athensCredentialsRequired(environment, tenant, applicationId, principal)) { ScrewdriverId screwdriverId = optionalScrewdriverId.orElseThrow( () -> loggedForbiddenException("Screwdriver id must be provided when deploying from Screwdriver.")); - principal = athens.principalFrom(screwdriverId); + principal = AthenzUtils.createPrincipal(screwdriverId); checkAthensCredentials(principal, tenant, applicationId); } } - private boolean hasDeployAccessToAthensApplication(AthensPrincipal principal, AthensDomain domain, ApplicationId applicationId) { + private boolean hasDeployAccessToAthensApplication(AthenzPrincipal principal, AthensDomain domain, ApplicationId applicationId) { try { - return athens.zmsClientFactory().createClientWithServicePrincipal() + return athenzClientFactory.createZmsClientWithServicePrincipal() .hasApplicationAccess( principal, ApplicationAction.deploy, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java index 0138d3ae65c..443a53b476d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/NTokenRequestFilter.java @@ -5,7 +5,7 @@ import com.google.inject.Inject; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; -import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; +import com.yahoo.vespa.hosted.controller.athenz.config.AthenzConfig; import com.yahoo.yolean.chain.After; /** @@ -16,16 +16,16 @@ public class NTokenRequestFilter implements SecurityRequestFilter { public static final String NTOKEN_HEADER = "com.yahoo.vespa.hosted.controller.restapi.filter.NTokenRequestFilter.ntoken"; - private final Athens athens; + private final String principalHeaderName; @Inject - public NTokenRequestFilter(Athens athens) { - this.athens = athens; + public NTokenRequestFilter(AthenzConfig config) { + this.principalHeaderName = config.principalHeaderName(); } @Override public void filter(DiscFilterRequest request, ResponseHandler responseHandler) { - String nToken = request.getHeader(athens.principalTokenHeader()); + String nToken = request.getHeader(principalHeaderName); if (nToken != null) { request.setAttribute(NTOKEN_HEADER, nToken); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java index fbd1a74c12c..6174a017a54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import java.util.List; +import java.util.Objects; /** * Statistics about deployments on a platform version. This is immutable. @@ -18,8 +19,9 @@ public class DeploymentStatistics { private final ImmutableList<ApplicationId> failing; private final ImmutableList<ApplicationId> production; - private DeploymentStatistics(Version version, - List<ApplicationId> failingApplications, List<ApplicationId> production) { + /** DO NOT USE. Public for serialization purposes */ + public DeploymentStatistics(Version version, List<ApplicationId> failingApplications, + List<ApplicationId> production) { this.version = version; this.failing = ImmutableList.copyOf(failingApplications); this.production = ImmutableList.copyOf(production); @@ -59,4 +61,18 @@ public class DeploymentStatistics { return b.build(); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DeploymentStatistics)) return false; + DeploymentStatistics that = (DeploymentStatistics) o; + return Objects.equals(version, that.version) && + Objects.equals(failing, that.failing) && + Objects.equals(production, that.production); + } + + @Override + public int hashCode() { + return Objects.hash(version, failing, production); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index c3f3773cde6..c3b2da42861 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -5,9 +5,9 @@ import com.google.common.collect.ImmutableList; import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.component.Vtag; -import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; @@ -46,7 +46,7 @@ public class VersionStatus { private final ImmutableList<VespaVersion> versions; - /** Create a version status. DO NOT USE: Public for testing only */ + /** Create a version status. DO NOT USE: Public for testing and serialization only */ public VersionStatus(List<VespaVersion> versions) { this.versions = ImmutableList.copyOf(versions); } @@ -175,11 +175,30 @@ public class VersionStatus { Collection<String> configServerHostnames, Controller controller) { GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString()); + Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime()); + VespaVersion.Confidence confidence; + // Always compute confidence for system version + if (isSystemVersion) { + confidence = VespaVersion.confidenceFrom(statistics, controller, releasedAt); + } else { + // Keep existing confidence for non-system versions if already computed + confidence = confidenceFor(statistics.version(), controller) + .orElse(VespaVersion.confidenceFrom(statistics, controller, releasedAt)); + } return new VespaVersion(statistics, - gitSha.sha, Instant.ofEpochMilli(gitSha.commit.author.date.getTime()), + gitSha.sha, releasedAt, isSystemVersion, configServerHostnames, - controller); + confidence + ); + } + + /** Returns the current confidence for the given version */ + private static Optional<VespaVersion.Confidence> confidenceFor(Version version, Controller controller) { + return controller.versionStatus().versions().stream() + .filter(v -> version.equals(v.versionNumber())) + .map(VespaVersion::confidence) + .findFirst(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index 3208f4d09c6..c1b9c045fbe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.versions; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.component.Vtag; -import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; @@ -12,6 +11,8 @@ import java.time.Instant; import java.util.Collection; import java.util.Set; +import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; + /** * Information about a particular Vespa version. * VespaVersions are identified by their version number and ordered by increasing version numbers. @@ -26,23 +27,22 @@ public class VespaVersion implements Comparable<VespaVersion> { private final Instant releasedAt; private final boolean isCurrentSystemVersion; private final DeploymentStatistics statistics; - private final Confidence confidence; private final ImmutableSet<String> configServerHostnames; + private final Confidence confidence; - public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant releasedAt, + public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant releasedAt, boolean isCurrentSystemVersion, Collection<String> configServerHostnames, - Controller controller) { + Confidence confidence) { this.statistics = statistics; this.releaseCommit = releaseCommit; this.releasedAt = releasedAt; this.isCurrentSystemVersion = isCurrentSystemVersion; this.configServerHostnames = ImmutableSet.copyOf(configServerHostnames); - this.confidence = deduceConfidenceFrom(statistics, controller, releasedAt); + this.confidence = confidence; } - private static Confidence deduceConfidenceFrom(DeploymentStatistics statistics, - Controller controller, - Instant releasedAt) { + public static Confidence confidenceFrom(DeploymentStatistics statistics, Controller controller, + Instant releasedAt) { // 'production on this': All deployment jobs upgrading to this version have completed without failure ApplicationList productionOnThis = ApplicationList.from(statistics.production(), controller.applications()) .notUpgradingTo(statistics.version()) diff --git a/controller-server/src/main/resources/configdefinitions/athenz.def b/controller-server/src/main/resources/configdefinitions/athenz.def new file mode 100644 index 00000000000..4e27e3ebd07 --- /dev/null +++ b/controller-server/src/main/resources/configdefinitions/athenz.def @@ -0,0 +1,26 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.controller.athenz.config + +# Principal header name +principalHeaderName string default="Athenz-Principal-Auth" + +# URL to ZMS API endpoint +zmsUrl string + +# URL to ZTS API endpoint +ztsUrl string + +# Athenz domain for controller identity. The domain is also used for Athenz tenancy integration. +domain string + +# Athenz service name for controller identity +service.name string + +# Athenz service public key id +service.publicKeyId string + +# Version of Athenz service private key +service.privateKeyVersion int + +# Name of Athenz service private key secret +service.privateKeySecretName string diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java index 9db852374b8..9228e83bbc6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java @@ -7,16 +7,16 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.rotation.Rotation; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.vespa.serviceview.bindings.ClusterView; @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -40,15 +39,11 @@ import java.util.UUID; */ public class ConfigServerClientMock extends AbstractComponent implements ConfigServerClient { - private Map<ApplicationId, byte[]> applicationContent = new HashMap<>(); - private Map<ApplicationId, String> applicationInstances = new HashMap<>(); - private Map<ApplicationId, Boolean> applicationActivated = new HashMap<>(); - private Set<ApplicationId> applicationRestarted = new HashSet<>(); - private Set<String> hostsExplicitlyRestarted = new HashSet<>(); - private Map<String, EndpointStatus> endpoints = new HashMap<>(); - - private Map<URI, Version> configServerVersions = new HashMap<>(); - private Version defaultConfigServerVersion = new Version(6, 1, 0); + private final Map<ApplicationId, String> applicationInstances = new HashMap<>(); + private final Map<ApplicationId, Boolean> applicationActivated = new HashMap<>(); + private final Map<String, EndpointStatus> endpoints = new HashMap<>(); + private final Map<URI, Version> versions = new HashMap<>(); + private Version defaultVersion = new Version(6, 1, 0); /** The exception to throw on the next prepare run, or null to continue normally */ private RuntimeException prepareException = null; @@ -64,18 +59,27 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS public Map<ApplicationId, Boolean> activated() { return Collections.unmodifiableMap(applicationActivated); } + + public void throwOnNextPrepare(RuntimeException prepareException) { + this.prepareException = prepareException; + } + + /** + * Returns the (initially empty) mutable map of config server urls to versions. + * This API will return defaultVersion as response to any version(url) call for versions not added to the map. + */ + public Map<URI, Version> versions() { + return versions; + } @Override public PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content) { lastPrepareVersion = deployOptions.vespaVersion.map(Version::new); - if (prepareException != null) { RuntimeException prepareException = this.prepareException; this.prepareException = null; throw prepareException; } - - applicationContent.put(deployment.applicationId(), content); applicationActivated.put(deployment.applicationId(), false); applicationInstances.put(deployment.applicationId(), UUID.randomUUID() + ":4080"); @@ -111,20 +115,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS }; } - public void throwOnNextPrepare(RuntimeException prepareException) { - this.prepareException = prepareException; - } - - /** - * Returns the (initially empty) mutable map of config server urls to versions. - * This API will return defaultConfigserverVersion as response to any version(url) call for versions not added to the map. - */ - public Map<URI, Version> configServerVersions() { - return configServerVersions; - } - - public Version getDefaultConfigServerVersion() { return defaultConfigServerVersion; } - public void setDefaultConfigServerVersion(Version version) { defaultConfigServerVersion = version; } + /** Set the default config server version */ + public void setDefaultVersion(Version version) { this.defaultVersion = version; } @Override public List<String> getNodeQueryHost(DeploymentId deployment, String type) { @@ -137,16 +129,11 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS @Override public void restart(DeploymentId deployment, Optional<Hostname> hostname) { - applicationRestarted.add(deployment.applicationId()); - if (hostname.isPresent()) { - hostsExplicitlyRestarted.add(hostname.get().id()); - } } @Override public void deactivate(DeploymentId deployment) { applicationActivated.remove(deployment.applicationId()); - applicationContent.remove(deployment.applicationId()); applicationInstances.remove(deployment.applicationId()); } @@ -197,7 +184,7 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS @Override public Version version(URI configServerURI) { - return configServerVersions.getOrDefault(configServerURI, defaultConfigServerVersion); + return versions.getOrDefault(configServerURI, defaultVersion); } @Override @@ -207,10 +194,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS @Override public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) { - EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757l); - return endpoints.containsKey(endpoint) - ? endpoints.get(endpoint) - : result; + EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757L); + return endpoints.getOrDefault(endpoint, result); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 6fc787d940e..408be6a49c2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -24,9 +24,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob; -import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.NTokenMock; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ApplicationRevision; @@ -36,6 +33,8 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthensDbMock; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildSystem; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -260,16 +259,19 @@ public class ControllerTest { Version systemVersion = controller.versionStatus().systemVersion().get().versionNumber(); Version newSystemVersion = new Version(systemVersion.getMajor(), systemVersion.getMinor()+1, 0); VespaVersion newSystemVespaVersion = new VespaVersion(DeploymentStatistics.empty(newSystemVersion), - "commit1", + "commit1", Instant.now(), true, Collections.emptyList(), - controller); + VespaVersion.Confidence.low + ); List<VespaVersion> versions = new ArrayList<>(controller.versionStatus().versions()); for (int i = 0; i < versions.size(); i++) { VespaVersion c = versions.get(i); if (c.isCurrentSystemVersion()) - versions.set(i, new VespaVersion(c.statistics(), c.releaseCommit(), c.releasedAt(), false, c.configServerHostnames(), controller)); + versions.set(i, new VespaVersion(c.statistics(), c.releaseCommit(), c.releasedAt(), + false, c.configServerHostnames(), + c.confidence())); } versions.add(newSystemVespaVersion); controller.updateVersionStatus(new VersionStatus(versions)); @@ -378,7 +380,7 @@ public class ControllerTest { assertFalse(mockDomain.isVespaTenant); // Migrate tenant to Athens - NToken nToken = new NTokenMock("token"); + NToken nToken = TestIdentities.userNToken; tester.controller().tenants().migrateTenantToAthens( tenantId, athensDomain, new PropertyId("1567"), new Property("vespa_dev.no"), nToken); @@ -542,7 +544,7 @@ public class ControllerTest { // Current system version, matches version in test data Version version = Version.fromString("6.141.117"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -572,7 +574,7 @@ public class ControllerTest { // New version is released version = Version.fromString("6.142.1"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); tester.upgrader().maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 5184cde79c2..8e1234b7e96 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -21,8 +21,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; import com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService; import com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService; @@ -30,6 +28,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock; import com.yahoo.vespa.hosted.controller.api.integration.jira.JiraMock; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthensDbMock; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.integration.MockMetricsService; import com.yahoo.vespa.hosted.controller.persistence.ControllerDb; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -214,7 +214,7 @@ public final class ControllerTester { new MockRoutingGenerator(), new ChefMock(), clock, - new AthensMock(athensDb)); + new AthenzClientFactoryMock(athensDb)); controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java index 1f52ebcadb7..355b63335c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/TestIdentities.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.NTokenMock; import com.yahoo.vespa.hosted.controller.api.Tenant; import com.yahoo.vespa.hosted.controller.api.identifiers.EnvironmentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; @@ -10,7 +9,9 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.RegionId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserGroup; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; +import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.filter.AthenzTestUtils; /** * @author Tony Vaagenes @@ -33,6 +34,8 @@ public class TestIdentities { public static Tenant tenant = Tenant.createOpsDbTenant(tenantId, userGroup1, property); - public static NToken userNToken = new NTokenMock("token"); + public static NToken userNToken = new NToken.Builder( + "U1", AthenzUtils.createPrincipal(userId), AthenzTestUtils.generateRsaKeypair().getPrivate(), "0") + .build(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java new file mode 100644 index 00000000000..313f565f546 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/ClusterCostTest.java @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ClusterSpec; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author smorgrav + */ +public class ClusterCostTest { + + @Test + public void clusterCost() throws Exception { + List<String> hostnames = new ArrayList<>(); + hostnames.add("host1"); + hostnames.add("host2"); + ClusterInfo info = new ClusterInfo("test", 100, 10, 10, 10, ClusterSpec.Type.container, hostnames); + ClusterUtilization util = new ClusterUtilization(0.3, 0.2, 0.5, 0.1); + ClusterCost cost = new ClusterCost(info, util); + + // CPU is fully utilized + Assert.assertEquals(200, cost.getTco(), Double.MIN_VALUE); + Assert.assertEquals(0, cost.getWaste(), Double.MIN_VALUE); + + // Set Disk as the most utilized resource + util = new ClusterUtilization(0.3, 0.1, 0.5, 0.1); + cost = new ClusterCost(info, util); + Assert.assertEquals(200, cost.getTco(), Double.MIN_NORMAL); // TCO is independent of utilization + Assert.assertEquals(57.1428571429, cost.getWaste(), 0.001); // Waste is not independent + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java new file mode 100644 index 00000000000..2e58253d768 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentCostTest.java @@ -0,0 +1,38 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ClusterSpec; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author smorgrav + */ +public class DeploymentCostTest { + + @Test + public void deploymentCost() { + Map<String, ClusterCost> clusters = new HashMap<>(); + clusters.put("cluster1", createClusterCost(100, 0.2)); + clusters.put("cluster2", createClusterCost(50, 0.1)); + + DeploymentCost cost = new DeploymentCost(clusters); + Assert.assertEquals(300, cost.getTco(), Double.MIN_VALUE); // 2*100 + 2*50 + Assert.assertEquals(28.5714285714, cost.getWaste(), 0.001); // from cluster2 + Assert.assertEquals(0.7142857142857143, cost.getUtilization(), Double.MIN_VALUE); // from cluster2 + } + + private ClusterCost createClusterCost(int flavorCost, double cpuUtil) { + List<String> hostnames = new ArrayList<>(); + hostnames.add("host1"); + hostnames.add("host2"); + ClusterInfo info = new ClusterInfo("test", flavorCost, 10, 10, 10, ClusterSpec.Type.container, hostnames); + ClusterUtilization util = new ClusterUtilization(0.3, cpuUtil, 0.5, 0.1); + return new ClusterCost(info, util); + } +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java new file mode 100644 index 00000000000..20db038485d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzPrincipalFilterTest.java @@ -0,0 +1,123 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.filter; + +import com.yahoo.jdisc.Response; +import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.jdisc.handler.ReadableContentChannel; +import com.yahoo.jdisc.handler.ResponseHandler; +import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import org.junit.Before; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.util.Objects; + +import static com.yahoo.jdisc.Response.Status.UNAUTHORIZED; +import static java.util.stream.Collectors.joining; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * @author bjorncs + */ +public class AthenzPrincipalFilterTest { + + private static final NToken NTOKEN = createDummyToken(); + private static final String ATHENZ_PRINCIPAL_HEADER = "Athenz-Principal-Auth"; + + private NTokenValidator validator; + private AthenzPrincipal principal; + + @Before + public void before() { + validator = mock(NTokenValidator.class); + principal = AthenzUtils.createPrincipal(new UserId("bob")); + } + + @Test + public void valid_ntoken_is_accepted() throws Exception { + DiscFilterRequest request = mock(DiscFilterRequest.class); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getToken()); + + when(validator.validate(NTOKEN)).thenReturn(principal); + + AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER); + filter.filter(request, new ResponseHandlerMock()); + + verify(request).setUserPrincipal(principal); + } + + @Test + public void missing_token_is_unauthorized() throws Exception { + DiscFilterRequest request = mock(DiscFilterRequest.class); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(null); + + ResponseHandlerMock responseHandler = new ResponseHandlerMock(); + + AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER); + filter.filter(request, responseHandler); + + assertThat(responseHandler.response, notNullValue()); + assertThat(responseHandler.response.getStatus(), equalTo(UNAUTHORIZED)); + assertThat(responseHandler.getResponseContent(), containsString("NToken is missing")); + } + + @Test + public void invalid_token_is_unauthorized() throws Exception { + DiscFilterRequest request = mock(DiscFilterRequest.class); + when(request.getHeader(ATHENZ_PRINCIPAL_HEADER)).thenReturn(NTOKEN.getToken()); + + when(validator.validate(NTOKEN)).thenThrow(new InvalidTokenException("Invalid token")); + + ResponseHandlerMock responseHandler = new ResponseHandlerMock(); + + AthenzPrincipalFilter filter = new AthenzPrincipalFilter(validator, Runnable::run, ATHENZ_PRINCIPAL_HEADER); + filter.filter(request, responseHandler); + + assertThat(responseHandler.response, notNullValue()); + assertThat(responseHandler.response.getStatus(), equalTo(UNAUTHORIZED)); + assertThat(responseHandler.getResponseContent(), containsString("Invalid token")); + } + + private static NToken createDummyToken() { + return new NToken.Builder( + "U1", AthenzUtils.createPrincipal(new UserId("bob")), AthenzTestUtils.generateRsaKeypair().getPrivate(), "0") + .build(); + } + + private static class ResponseHandlerMock implements ResponseHandler { + + public Response response; + public ReadableContentChannel contentChannel; + + @Override + public ContentChannel handleResponse(Response r) { + response = Objects.requireNonNull(r); + contentChannel = new ReadableContentChannel(); + return contentChannel; + } + + public String getResponseContent() { + try (BufferedReader br = new BufferedReader(new InputStreamReader(contentChannel.toStream()))) { + return br.lines().collect(joining(System.lineSeparator())); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTestUtils.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTestUtils.java new file mode 100644 index 00000000000..40b38254dda --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/AthenzTestUtils.java @@ -0,0 +1,22 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.filter; + +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; + +/** + * @author bjorncs + */ +public class AthenzTestUtils { + public static KeyPair generateRsaKeypair() { + try { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA"); + keyGen.initialize(512); + return keyGen.genKeyPair(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java new file mode 100644 index 00000000000..e269f2842e2 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/athenz/filter/NTokenValidatorTest.java @@ -0,0 +1,95 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.athenz.filter; + +import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; +import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.InvalidTokenException; +import com.yahoo.vespa.hosted.controller.athenz.NToken; +import com.yahoo.vespa.hosted.controller.athenz.ZmsKeystore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.security.KeyPair; +import java.security.NoSuchAlgorithmException; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.athenz.AthenzUtils.ZMS_ATHENZ_SERVICE; +import static org.junit.Assert.assertEquals; + +/** + * @author bjorncs + */ +public class NTokenValidatorTest { + + private static final KeyPair TRUSTED_KEY = AthenzTestUtils.generateRsaKeypair(); + private static final KeyPair UNKNOWN_KEY = AthenzTestUtils.generateRsaKeypair(); + private static final AthenzPrincipal PRINCIPAL = new AthenzPrincipal(new AthensDomain("yby"), new UserId("user")); + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void valid_token_is_accepted() throws NoSuchAlgorithmException, InvalidTokenException { + NTokenValidator validator = new NTokenValidator(createKeystore()); + NToken token = createNToken(PRINCIPAL, System.currentTimeMillis(), TRUSTED_KEY, "0"); + AthenzPrincipal principal = validator.validate(token); + assertEquals("yby.user", principal.toYRN()); + } + + @Test + public void invalid_signature_is_not_accepted() throws InvalidTokenException { + NTokenValidator validator = new NTokenValidator(createKeystore()); + NToken token = createNToken(PRINCIPAL, System.currentTimeMillis(), UNKNOWN_KEY, "0"); + exceptionRule.expect(InvalidTokenException.class); + exceptionRule.expectMessage("NToken is expired or has invalid signature"); + validator.validate(token); + } + + @Test + public void expired_token_is_not_accepted() throws InvalidTokenException { + NTokenValidator validator = new NTokenValidator(createKeystore()); + NToken token = createNToken(PRINCIPAL, 1234 /*long time ago*/, TRUSTED_KEY, "0"); + exceptionRule.expect(InvalidTokenException.class); + exceptionRule.expectMessage("NToken is expired or has invalid signature"); + validator.validate(token); + } + + @Test + public void unknown_keyId_is_not_accepted() throws InvalidTokenException { + NTokenValidator validator = new NTokenValidator(createKeystore()); + NToken token = createNToken(PRINCIPAL, System.currentTimeMillis(), TRUSTED_KEY, "unknown-key-id"); + exceptionRule.expect(InvalidTokenException.class); + exceptionRule.expectMessage("NToken has an unknown keyId"); + validator.validate(token); + } + + @Test + public void failing_to_find_key_should_throw_exception() throws InvalidTokenException { + ZmsKeystore keystore = (athensService, keyId) -> { throw new RuntimeException(); }; + NTokenValidator validator = new NTokenValidator(keystore); + NToken token = createNToken(PRINCIPAL, System.currentTimeMillis(), TRUSTED_KEY, "0"); + exceptionRule.expect(InvalidTokenException.class); + exceptionRule.expectMessage("Failed to retrieve public key"); + validator.validate(token); + } + + private static ZmsKeystore createKeystore() { + return (athensService, keyId) -> + athensService.equals(ZMS_ATHENZ_SERVICE) && keyId.equals("0") + ? Optional.of(TRUSTED_KEY.getPublic()) + : Optional.empty(); + } + + private static NToken createNToken(AthenzPrincipal principal, long issueTime, KeyPair keyPair, String keyId) { + return new NToken.Builder("U1", principal, keyPair.getPrivate(), keyId) + .salt("1234") + .hostname("host") + .ip("1.2.3.4") + .issueTime(issueTime / 1000) + .expirationWindow(1000) + .build(); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index be14947de2b..e45166b1c16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -90,12 +90,17 @@ public class DeploymentTester { .filter(c -> c instanceof Change.VersionChange) .map(Change.VersionChange.class::cast); } + + public void updateVersionStatus() { + controller().updateVersionStatus(VersionStatus.compute(controller(), tester.controller().systemVersion())); + } public void updateVersionStatus(Version currentVersion) { controller().updateVersionStatus(VersionStatus.compute(controller(), currentVersion)); } public void upgradeSystem(Version version) { + controllerTester().configServer().setDefaultVersion(version); updateVersionStatus(version); upgrader().maintain(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java index 7ae89082660..13919cefd3b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainerTest.java @@ -31,7 +31,7 @@ public class ClusterInfoMaintainerTest { deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); Assert.assertEquals(2, deployment.clusterInfo().size()); - Assert.assertEquals(10, deployment.clusterInfo().get(ClusterSpec.Id.from("clusterA")).getCost()); + Assert.assertEquals(10, deployment.clusterInfo().get(ClusterSpec.Id.from("clusterA")).getFlavorCost()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java new file mode 100644 index 00000000000..cb503fc1fb7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -0,0 +1,41 @@ +package com.yahoo.vespa.hosted.controller.maintenance; + +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import org.junit.Assert; +import org.junit.Test; + +import java.time.Duration; + +/** + * @author smorgrav + */ +public class DeploymentMetricsMaintainerTest { + + @Test + public void maintain() { + ControllerTester tester = new ControllerTester(); + ApplicationId app = tester.createAndDeploy("tenant1", "domain1", "app1", Environment.dev, 123).id(); + + // Pre condition: no metric info on the deployment + Deployment deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); + Assert.assertEquals(0, deployment.metrics().documentCount(), Double.MIN_VALUE); + + DeploymentMetricsMaintainer maintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofMinutes(10), new JobControl(new MockCuratorDb())); + maintainer.maintain(); + + // Post condition: + deployment = tester.controller().applications().get(app).get().deployments().values().stream().findAny().get(); + Assert.assertEquals(1, deployment.metrics().queriesPerSecond(), Double.MIN_VALUE); + Assert.assertEquals(2, deployment.metrics().writesPerSecond(), Double.MIN_VALUE); + Assert.assertEquals(3, deployment.metrics().documentCount(), Double.MIN_VALUE); + Assert.assertEquals(4, deployment.metrics().queryLatencyMillis(), Double.MIN_VALUE); + Assert.assertEquals(5, deployment.metrics().writeLatencyMillis(), Double.MIN_VALUE); + } + +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java index 286db864c22..4c935747ac6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java @@ -181,7 +181,7 @@ public class FailureRedeployerTest { // Current system version, matches version in test data Version version = Version.fromString("6.141.117"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -200,7 +200,7 @@ public class FailureRedeployerTest { // New version is released version = Version.fromString("6.142.1"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); tester.upgrader().maintain(); @@ -237,7 +237,7 @@ public class FailureRedeployerTest { // Current system version, matches version in test data Version version = Version.fromString("6.42.1"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); @@ -262,7 +262,7 @@ public class FailureRedeployerTest { // Current system version, matches version in test data Version version = Version.fromString("6.42.1"); - tester.configServer().setDefaultConfigServerVersion(version); + tester.configServer().setDefaultVersion(version); tester.updateVersionStatus(version); assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 3e73bf4445b..f7fd7cb6fb1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.SourceRevision; import org.junit.Test; @@ -61,7 +62,7 @@ public class ApplicationSerializerTest { ApplicationRevision revision2 = ApplicationRevision.from("appHash2", new SourceRevision("repo1", "branch1", "commit1")); deployments.add(new Deployment(zone1, revision1, Version.fromString("1.2.3"), Instant.ofEpochMilli(3))); // One deployment without cluster info and utils deployments.add(new Deployment(zone2, revision2, Version.fromString("1.2.3"), Instant.ofEpochMilli(5), - createClusterUtils(3, 0.2), createClusterInfo(3, 4))); + createClusterUtils(3, 0.2), createClusterInfo(3, 4),new DeploymentMetrics(2,3,4,5,6))); Optional<Long> projectId = Optional.of(123L); List<JobStatus> statusList = new ArrayList<>(); @@ -118,10 +119,20 @@ public class ApplicationSerializerTest { // Test cluster info assertEquals(3, serialized.deployments().get(zone2).clusterInfo().size()); - assertEquals(10, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getCost()); + assertEquals(10, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCost()); assertEquals(ClusterSpec.Type.content, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getClusterType()); assertEquals("flavor2", serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavor()); assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getHostnames().size()); + assertEquals(2, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorCPU(), Double.MIN_VALUE); + assertEquals(4, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorMem(), Double.MIN_VALUE); + assertEquals(50, serialized.deployments().get(zone2).clusterInfo().get(ClusterSpec.Id.from("id2")).getFlavorDisk(), Double.MIN_VALUE); + + // Test metrics + assertEquals(2, serialized.deployments().get(zone2).metrics().queriesPerSecond(), Double.MIN_VALUE); + assertEquals(3, serialized.deployments().get(zone2).metrics().writesPerSecond(), Double.MIN_VALUE); + assertEquals(4, serialized.deployments().get(zone2).metrics().documentCount(), Double.MIN_VALUE); + assertEquals(5, serialized.deployments().get(zone2).metrics().queryLatencyMillis(), Double.MIN_VALUE); + assertEquals(6, serialized.deployments().get(zone2).metrics().writeLatencyMillis(), Double.MIN_VALUE); { // test more deployment serialization cases Application original2 = original.withDeploying(Optional.of(Change.ApplicationChange.of(ApplicationRevision.from("hash1")))); @@ -153,7 +164,7 @@ public class ApplicationSerializerTest { } result.put(ClusterSpec.Id.from("id" + cluster), new ClusterInfo("flavor" + cluster, 10, - ClusterSpec.Type.content, hostnames)); + 2, 4, 50, ClusterSpec.Type.content, hostnames)); } return result; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java new file mode 100644 index 00000000000..feaf1289853 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -0,0 +1,53 @@ +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; +import org.junit.Test; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class VersionStatusSerializerTest { + + @Test + public void testSerialization() throws Exception { + List<VespaVersion> vespaVersions = new ArrayList<>(); + DeploymentStatistics statistics = new DeploymentStatistics( + Version.fromString("5.0"), + Arrays.asList(ApplicationId.from("tenant1", "failing1", "default")), + Arrays.asList(ApplicationId.from("tenant2", "success1", "default"), + ApplicationId.from("tenant2", "success2", "default")) + ); + vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, + Arrays.asList("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, + Arrays.asList("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + VersionStatus status = new VersionStatus(vespaVersions); + VersionStatusSerializer serializer = new VersionStatusSerializer(); + VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status)); + + assertEquals(status.versions().size(), deserialized.versions().size()); + for (int i = 0; i < status.versions().size(); i++) { + VespaVersion a = status.versions().get(i); + VespaVersion b = deserialized.versions().get(i); + assertEquals(a.releaseCommit(), b.releaseCommit()); + assertEquals(a.releasedAt(), b.releasedAt()); + assertEquals(a.isCurrentSystemVersion(), b.isCurrentSystemVersion()); + assertEquals(a.statistics(), b.statistics()); + assertEquals(a.configServerHostnames(), b.configServerHostnames()); + assertEquals(a.confidence(), b.confidence()); + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index a792626d691..99381d538d5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -21,13 +21,11 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; -import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthensDbMock; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -97,14 +95,12 @@ public class ContainerControllerTester { } public AthensDomain addTenantAthensDomain(String domainName, String userName) { - Athens athens = (AthensMock) containerTester.container().components().getComponent( - "com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock" - ); - ZmsClientFactoryMock mock = (ZmsClientFactoryMock) athens.zmsClientFactory(); + AthenzClientFactoryMock mock = (AthenzClientFactoryMock) containerTester.container().components() + .getComponent(AthenzClientFactoryMock.class.getName()); AthensDomain athensDomain = new AthensDomain(domainName); AthensDbMock.Domain domain = new AthensDbMock.Domain(athensDomain); domain.markAsVespaTenant(); - domain.admin(new AthensPrincipal(new AthensDomain("domain"), new UserId(userName))); + domain.admin(new AthenzPrincipal(new AthensDomain("domain"), new UserId(userName))); mock.getSetup().addDomain(domain); return athensDomain; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index fd07428126a..5c102b0d9df 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -36,7 +36,7 @@ public class ControllerContainerTest { " <system>main</system>" + " </config>" + " <component id='com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb'/>" + - " <component id='com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock'/>" + + " <component id='com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock'/>" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.dns.MemoryNameService'/>" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.entity.MemoryEntityService'/>" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 13b1165ccb2..0c9ebedc09b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -10,17 +10,17 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ConfigServerClientMock; import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens; -import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock; -import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.AthenzUtils; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthensDbMock; +import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -54,7 +54,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .region("corp-us-east-1") .build(); private static final String athensUserDomain = "domain1"; - private static final String athensScrewdriverDomain = "screwdriver-domain"; + private static final String athensScrewdriverDomain = AthenzUtils.SCREWDRIVER_DOMAIN.id(); @Test @@ -173,7 +173,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET), new File("application.json")); // GET an application deployment - addMockObservedApplicationCost(controllerTester); + setDeploymentMaintainedInfo(controllerTester); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/corp-us-east-1/instance/default", "", Request.Method.GET), new File("deployment.json")); // POST a 'restart application' command @@ -671,14 +671,12 @@ public class ApplicationApiTest extends ControllerContainerTest { * mock setup to replicate the action. */ private AthensDomain addTenantAthensDomain(String domainName, String userName) { - Athens athens = (AthensMock) container.components().getComponent( - "com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock" - ); - ZmsClientFactoryMock mock = (ZmsClientFactoryMock) athens.zmsClientFactory(); + AthenzClientFactoryMock mock = (AthenzClientFactoryMock) container.components() + .getComponent(AthenzClientFactoryMock.class.getName()); AthensDomain athensDomain = new AthensDomain(domainName); AthensDbMock.Domain domain = new AthensDbMock.Domain(athensDomain); domain.markAsVespaTenant(); - domain.admin(new AthensPrincipal(new AthensDomain(athensUserDomain), new UserId(userName))); + domain.admin(AthenzUtils.createPrincipal(new UserId(userName))); mock.getSetup().addDomain(domain); return athensDomain; } @@ -688,12 +686,10 @@ public class ApplicationApiTest extends ControllerContainerTest { * mock setup to replicate the action. */ private void addScrewdriverUserToDomain(String screwdriverUserId, String domainName) { - Athens athens = (AthensMock) container.components().getComponent( - "com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock" - ); - ZmsClientFactoryMock mock = (ZmsClientFactoryMock) athens.zmsClientFactory(); + AthenzClientFactoryMock mock = (AthenzClientFactoryMock) container.components() + .getComponent(AthenzClientFactoryMock.class.getName()); AthensDbMock.Domain domain = mock.getSetup().domains.get(new AthensDomain(domainName)); - domain.admin(new AthensPrincipal(new AthensDomain(athensScrewdriverDomain), new UserId(screwdriverUserId))); + domain.admin(new AthenzPrincipal(new AthensDomain(athensScrewdriverDomain), new UserId(screwdriverUserId))); } private void startAndTestChange(ContainerControllerTester controllerTester, ApplicationId application, long projectId, @@ -732,7 +728,14 @@ public class ApplicationApiTest extends ControllerContainerTest { controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.stagingTest); } - private void addMockObservedApplicationCost(ContainerControllerTester controllerTester) { + /** + * Cluster info, utilization and deployment metrics are maintained async by maintainers. + * + * This sets these values as if the maintainers has been ran. + * + * @param controllerTester + */ + private void setDeploymentMaintainedInfo(ContainerControllerTester controllerTester) { for (Application application : controllerTester.controller().applications().asList()) { try (Lock lock = controllerTester.controller().applications().lock(application.id())) { for (Deployment deployment : application.deployments().values()) { @@ -740,11 +743,12 @@ public class ApplicationApiTest extends ControllerContainerTest { List<String> hostnames = new ArrayList<>(); hostnames.add("host1"); hostnames.add("host2"); - clusterInfo.put(ClusterSpec.Id.from("cluster1"), new ClusterInfo("flavor1", 37, ClusterSpec.Type.content, hostnames)); + clusterInfo.put(ClusterSpec.Id.from("cluster1"), new ClusterInfo("flavor1", 37, 2, 4, 50, ClusterSpec.Type.content, hostnames)); Map<ClusterSpec.Id, ClusterUtilization> clusterUtils = new HashMap<>(); clusterUtils.put(ClusterSpec.Id.from("cluster1"), new ClusterUtilization(0.3, 0.6, 0.4, 0.3)); deployment = deployment.withClusterInfo(clusterInfo); deployment = deployment.withClusterUtils(clusterUtils); + deployment = deployment.withMetrics(new DeploymentMetrics(1,2,3,4,5)); application = application.with(deployment); controllerTester.controller().applications().store(application, lock); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java index 16557157cf5..6f8dfc681ac 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/MockAuthorizer.java @@ -6,9 +6,10 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TestIdentities; import com.yahoo.vespa.hosted.controller.api.identifiers.AthensDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.UserId; -import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal; -import com.yahoo.vespa.hosted.controller.api.integration.athens.NToken; import com.yahoo.vespa.hosted.controller.api.integration.entity.EntityService; +import com.yahoo.vespa.hosted.controller.athenz.AthenzClientFactory; +import com.yahoo.vespa.hosted.controller.athenz.AthenzPrincipal; +import com.yahoo.vespa.hosted.controller.athenz.NToken; import javax.ws.rs.core.SecurityContext; import java.security.Principal; @@ -23,15 +24,15 @@ import java.util.Optional; @SuppressWarnings("unused") // injected public class MockAuthorizer extends Authorizer { - public MockAuthorizer(Controller controller, EntityService entityService) { - super(controller, entityService); + public MockAuthorizer(Controller controller, EntityService entityService, AthenzClientFactory athenzClientFactory) { + super(controller, entityService, athenzClientFactory); } /** Returns a principal given by the request parameters 'domain' and 'user' */ @Override public Optional<Principal> getPrincipalIfAny(HttpRequest request) { if (request.getProperty("user") == null) return Optional.empty(); - return Optional.of(new AthensPrincipal(new AthensDomain(request.getProperty("domain")), + return Optional.of(new AthenzPrincipal(new AthensDomain(request.getProperty("domain")), new UserId(request.getProperty("user")))); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 67fc48d4646..9174e7dd8b2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -17,17 +17,21 @@ "gitBranch": "master", "gitCommit": "commit1", "cost": { - "tco": 37, - "utilization": 5.999999999999999, - "waste": 0.0, + "tco": 74, + "waste": 0, + "utilization": 2.999999999999999, "cluster": { "cluster1": { "count": 2, "resource": "cpu", "utilization": 2.999999999999999, - "tco": 37, - "flavor": "flavor1", + "tco": 74, "waste": 0, + "flavor": "flavor1", + "flavorCost":37.0, + "flavorCpu":2.0, + "flavorMem":4.0, + "flavorDisk":50.0, "type": "content", "util": { "cpu": 2.999999999999999, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index e974c315eb2..3633860772b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -4,10 +4,7 @@ "name": "DelayedDeployer" }, { - "name": "ClusterUtilizationMaintainer" - }, - { - "name":"BlockedChangeDeployer" + "name": "BlockedChangeDeployer" }, { "name": "Upgrader" @@ -16,25 +13,31 @@ "name": "FailureRedeployer" }, { - "name": "ClusterInfoMaintainer" + "name": "VersionStatusUpdater" }, { - "name": "DeploymentExpirer" + "name": "DeploymentIssueReporter" }, { - "name": "MetricsReporter" + "name": "DeploymentMetricsMaintainer" }, { - "name": "VersionStatusUpdater" + "name": "OutstandingChangeDeployer" }, { - "name": "DeploymentIssueReporter" + "name": "ClusterUtilizationMaintainer" }, { - "name": "OutstandingChangeDeployer" + "name": "ClusterInfoMaintainer" + }, + { + "name": "DeploymentExpirer" + }, + { + "name": "MetricsReporter" } ], "inactive": [ "DeploymentExpirer" ] -} +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index c002c7fb24a..55a4b46f4a7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -85,7 +85,8 @@ public class DeploymentApiTest extends ControllerContainerTest { version.releasedAt(), version.isCurrentSystemVersion(), ImmutableSet.of("config1.test", "config2.test"), - controller); + VespaVersion.confidenceFrom(version.statistics(), controller, version.releasedAt()) + ); censored.add(version); } return new VersionStatus(censored); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index c2e83373cf7..00bd1ed8208 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -2,7 +2,7 @@ "versions": [ { "version": "(ignore)", - "confidence": "normal", + "confidence": "high", "commit": "(ignore)", "date": 0, "controllerVersion": false, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 7bbbf8f0499..17935906186 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -6,6 +6,7 @@ import com.yahoo.component.Vtag; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import org.junit.Test; import java.net.URI; @@ -46,7 +48,7 @@ public class VersionStatusTest { public void testSystemVersionIsControllerVersionIfConfigserversAreNewer() { ControllerTester tester = new ControllerTester(); Version largerThanCurrent = new Version(Vtag.currentVersion.getMajor() + 1); - tester.configServer().setDefaultConfigServerVersion(largerThanCurrent); + tester.configServer().setDefaultVersion(largerThanCurrent); VersionStatus versionStatus = VersionStatus.compute(tester.controller()); assertEquals(Vtag.currentVersion, versionStatus.systemVersion().get().versionNumber()); } @@ -55,7 +57,7 @@ public class VersionStatusTest { public void testSystemVersionIsVersionOfOldestConfigServer() throws URISyntaxException { ControllerTester tester = new ControllerTester(); Version oldest = new Version(5); - tester.configServer().configServerVersions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest); + tester.configServer().versions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest); VersionStatus versionStatus = VersionStatus.compute(tester.controller()); assertEquals(oldest, versionStatus.systemVersion().get().versionNumber()); } @@ -70,7 +72,6 @@ public class VersionStatusTest { .region("us-east-3") .build(); - // Application versions which are older than the current version Version version1 = new Version("5.1"); Version version2 = new Version("5.2"); tester.upgradeSystem(version1); @@ -90,14 +91,9 @@ public class VersionStatusTest { // - app3 is in production on version1, but then fails in staging test on version2 tester.completeUpgradeWithError(app3, version2, applicationPackage, stagingTest); - VersionStatus versionStatus = VersionStatus.compute(tester.controller()); - List<VespaVersion> versions = versionStatus.versions(); - assertEquals("The version of this controller, the default config server version, plus the two versions above exist", 4, versions.size()); - - VespaVersion v0 = versions.get(2); - assertEquals(tester.configServer().getDefaultConfigServerVersion(), v0.versionNumber()); - assertEquals(0, v0.statistics().failing().size()); - assertEquals(0, v0.statistics().production().size()); + tester.updateVersionStatus(); + List<VespaVersion> versions = tester.controller().versionStatus().versions(); + assertEquals("The two versions above exist", 2, versions.size()); VespaVersion v1 = versions.get(0); assertEquals(version1, v1.versionNumber()); @@ -116,11 +112,6 @@ public class VersionStatusTest { // Only one application is on v2 in at least one zone assertEquals(1, v2.statistics().production().size()); assertTrue(v2.statistics().production().contains(app2.id())); - - VespaVersion v3 = versions.get(3); - assertEquals(Vtag.currentVersion, v3.versionNumber()); - assertEquals(0, v3.statistics().failing().size()); - assertEquals(0, v3.statistics().production().size()); } @Test @@ -130,7 +121,7 @@ public class VersionStatusTest { Version version0 = new Version("5.0"); tester.upgradeSystem(version0); - // Setup applications + // Setup applications - all running on version0 Application canary0 = tester.createAndDeploy("canary0", 1, "canary"); Application canary1 = tester.createAndDeploy("canary1", 2, "canary"); Application canary2 = tester.createAndDeploy("canary2", 3, "canary"); @@ -146,8 +137,7 @@ public class VersionStatusTest { Application default9 = tester.createAndDeploy("default9", 13, "default"); Application conservative0 = tester.createAndDeploy("conservative1", 14, "conservative"); - - // The following applications should not affect confidence calculation: + // Applications that do not affect confidence calculation: // Application without deployment Application ignored0 = tester.createApplication("ignored0", "tenant1", 1000, 1000L); @@ -157,30 +147,40 @@ public class VersionStatusTest { "ignored1", "default-pr42", 1000); + assertEquals("All applications running on this version: High", + Confidence.high, confidence(tester.controller(), version0)); + + // New version is released Version version1 = new Version("5.1"); - Version version2 = new Version("5.2"); tester.upgradeSystem(version1); // Canaries upgrade to new versions and fail tester.completeUpgrade(canary0, version1, "canary"); tester.completeUpgradeWithError(canary1, version1, "canary", productionUsWest1); - tester.upgradeSystem(version2); - tester.completeUpgrade(canary2, version2, "canary"); - - VersionStatus versionStatus = VersionStatus.compute(tester.controller()); - List<VespaVersion> versions = versionStatus.versions(); - + tester.updateVersionStatus(); assertEquals("One canary failed: Broken", - VespaVersion.Confidence.broken, confidence(versions, version1)); - assertEquals("Nothing has failed but not all canaries has deployed: Low", - VespaVersion.Confidence.low, confidence(versions, version2)); - assertEquals("Current version of this - no deployments: Low", - VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + Confidence.broken, confidence(tester.controller(), version1)); - // All canaries are upgraded to version2 which raises confidence to normal and more apps upgrade + // New version is released + Version version2 = new Version("5.2"); + tester.upgradeSystem(version2); + assertEquals("Confidence defaults to low for version with no applications", + Confidence.low, confidence(tester.controller(), version2)); + + // All canaries upgrade successfully tester.completeUpgrade(canary0, version2, "canary"); tester.completeUpgrade(canary1, version2, "canary"); + + assertEquals("Confidence for remains unchanged for version1: Broken", + Confidence.broken, confidence(tester.controller(), version1)); + assertEquals("Nothing has failed but not all canaries have upgraded: Low", + Confidence.low, confidence(tester.controller(), version2)); + + // Remaining canary upgrades to version2 which raises confidence to normal and more apps upgrade + tester.completeUpgrade(canary2, version2, "canary"); tester.upgradeSystem(version2); + assertEquals("Canaries have upgraded: Normal", + Confidence.normal, confidence(tester.controller(), version2)); tester.completeUpgrade(default0, version2, "default"); tester.completeUpgrade(default1, version2, "default"); tester.completeUpgrade(default2, version2, "default"); @@ -189,29 +189,27 @@ public class VersionStatusTest { tester.completeUpgrade(default5, version2, "default"); tester.completeUpgrade(default6, version2, "default"); tester.completeUpgrade(default7, version2, "default"); + tester.updateVersionStatus(); - versionStatus = VersionStatus.compute(tester.controller()); - versions = versionStatus.versions(); + // Remember confidence across restart + tester.restartController(); - assertEquals("No deployments: Low", - VespaVersion.Confidence.low, confidence(versions, version0)); + assertEquals("Confidence remains unchanged for version0: High", + Confidence.high, confidence(tester.controller(), version0)); assertEquals("All canaries deployed + < 90% of defaults: Normal", - VespaVersion.Confidence.normal, confidence(versions, version2)); - assertEquals("Current version of this - no deployments: Low", - VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + Confidence.normal, confidence(tester.controller(), version2)); + assertTrue("Status for version without applications is removed", + tester.controller().versionStatus().versions().stream() + .noneMatch(vespaVersion -> vespaVersion.versionNumber().equals(version1))); // Another default application upgrades, raising confidence to high tester.completeUpgrade(default8, version2, "default"); + tester.updateVersionStatus(); - versionStatus = VersionStatus.compute(tester.controller()); - versions = versionStatus.versions(); - - assertEquals("No deployments: Low", - VespaVersion.Confidence.low, confidence(versions, version0)); + assertEquals("Confidence remains unchanged for version0: High", + Confidence.high, confidence(tester.controller(), version0)); assertEquals("90% of defaults deployed successfully: High", - VespaVersion.Confidence.high, confidence(versions, version2)); - assertEquals("Current version of this - no deployments: Low", - VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + VespaVersion.Confidence.high, confidence(tester.controller(), version2)); // A new version is released, all canaries upgrade successfully, but enough "default" apps fail to mark version // as broken @@ -225,16 +223,14 @@ public class VersionStatusTest { tester.completeUpgradeWithError(default1, version3, "default", stagingTest); tester.completeUpgradeWithError(default2, version3, "default", stagingTest); tester.completeUpgradeWithError(default9, version3, "default", stagingTest); + tester.updateVersionStatus(); - versionStatus = VersionStatus.compute(tester.controller()); - versions = versionStatus.versions(); - - assertEquals("No deployments: Low", - VespaVersion.Confidence.low, confidence(versions, version0)); + assertEquals("Confidence remains unchanged for version0: High", + Confidence.high, confidence(tester.controller(), version0)); + assertEquals("Confidence remains unchanged for version2: High", + Confidence.high, confidence(tester.controller(), version2)); assertEquals("40% of defaults failed: Broken", - VespaVersion.Confidence.broken, confidence(versions, version3)); - assertEquals("Current version of this - no deployments: Low", - VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion)); + VespaVersion.Confidence.broken, confidence(tester.controller(), version3)); } @Test @@ -260,8 +256,8 @@ public class VersionStatusTest { vespaVersions.stream().noneMatch(v -> v.versionNumber().equals(versionWithUnknownTag))); } - private VespaVersion.Confidence confidence(List<VespaVersion> versions, Version version) { - return versions.stream() + private Confidence confidence(Controller controller, Version version) { + return controller.versionStatus().versions().stream() .filter(v -> v.statistics().version().equals(version)) .findFirst() .map(VespaVersion::confidence) |