diff options
Diffstat (limited to 'controller-server/src/main/java')
72 files changed, 1749 insertions, 1097 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 84e15deea4c..d8b56502fc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -11,13 +11,16 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -56,8 +59,9 @@ public class Application { private final OptionalInt majorVersion; private final ApplicationMetrics metrics; private final Optional<String> pemDeployKey; - private final Optional<RotationId> rotation; + private final List<AssignedRotation> rotations; private final Map<HostName, RotationStatus> rotationStatus; + private final Optional<ApplicationCertificate> applicationCertificate; /** Creates an empty application */ public Application(ApplicationId id, Instant now) { @@ -65,7 +69,7 @@ public class Application { new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false), Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(0, 0), - Optional.empty(), Optional.empty(), Collections.emptyMap()); + Optional.empty(), Collections.emptyList(), Collections.emptyMap(), Optional.empty()); } /** Used from persistence layer: Do not use */ @@ -73,18 +77,19 @@ public class Application { List<Deployment> deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey, - Optional<RotationId> rotation, Map<HostName, RotationStatus> rotationStatus) { + List<AssignedRotation> rotations, Map<HostName, RotationStatus> rotationStatus, + Optional<ApplicationCertificate> applicationCertificate) { this(id, createdAt, deploymentSpec, validationOverrides, deployments.stream().collect(Collectors.toMap(Deployment::zone, Function.identity())), deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } Application(ApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey, - Optional<RotationId> rotation, Map<HostName, RotationStatus> rotationStatus) { + List<AssignedRotation> rotations, Map<HostName, RotationStatus> rotationStatus, Optional<ApplicationCertificate> applicationCertificate) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -98,8 +103,9 @@ public class Application { this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null"); this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); this.pemDeployKey = pemDeployKey; - this.rotation = Objects.requireNonNull(rotation, "rotation cannot be null"); + this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null")); this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null")); + this.applicationCertificate = Objects.requireNonNull(applicationCertificate, "applicationCertificate cannot be null"); } public ApplicationId id() { return id; } @@ -195,14 +201,36 @@ public class Application { } /** Returns the global rotation id of this, if present */ - public Optional<RotationId> rotation() { - return rotation; + public Optional<RotationId> legacyRotation() { + return rotations.stream() + .map(AssignedRotation::rotationId) + .findFirst(); + } + + /** Returns all rotations for this application */ + public List<RotationId> rotations() { + return rotations.stream() + .map(AssignedRotation::rotationId) + .collect(Collectors.toList()); + } + + /** Returns all assigned rotations for this application */ + public List<AssignedRotation> assignedRotations() { + return rotations; + } + + /** Returns the default global endpoints for this in given system - for a given endpoint ID */ + public EndpointList endpointsIn(SystemName system, EndpointId endpointId) { + if (rotations.isEmpty()) return EndpointList.EMPTY; + return EndpointList.create(id, endpointId, system); } /** Returns the default global endpoints for this in given system */ public EndpointList endpointsIn(SystemName system) { - if (rotation.isEmpty()) return EndpointList.EMPTY; - return EndpointList.defaultGlobal(id, system); + if (rotations.isEmpty()) return EndpointList.EMPTY; + final var endpointStream = rotations.stream() + .flatMap(rotation -> EndpointList.create(id, rotation.endpointId(), system).asList().stream()); + return EndpointList.of(endpointStream); } public Optional<String> pemDeployKey() { return pemDeployKey; } @@ -224,6 +252,10 @@ public class Application { .orElse(RotationStatus.unknown); } + public Optional<ApplicationCertificate> applicationCertificate() { + return applicationCertificate; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index b88f1bbb8e1..60fd095eb04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -7,7 +7,9 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -25,8 +27,11 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; @@ -36,25 +41,28 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.maintenance.RoutingPolicies; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.rotation.Rotation; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -75,20 +83,23 @@ import java.security.Principal; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; @@ -115,10 +126,12 @@ public class ApplicationController { private final AccessControl accessControl; private final ConfigServer configServer; private final RoutingGenerator routingGenerator; + private final RoutingPolicies routingPolicies; private final Clock clock; - private final BooleanFlag redirectLegacyDnsFlag; - + private final BooleanFlag useMultipleEndpoints; private final DeploymentTrigger deploymentTrigger; + private final BooleanFlag provisionApplicationCertificate; + private final ApplicationCertificateProvider applicationCertificateProvider; ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, RotationsConfig rotationsConfig, @@ -130,14 +143,18 @@ public class ApplicationController { this.accessControl = accessControl; this.configServer = configServer; this.routingGenerator = routingGenerator; + this.routingPolicies = new RoutingPolicies(controller); this.clock = clock; - this.redirectLegacyDnsFlag = Flags.REDIRECT_LEGACY_DNS_NAMES.bindTo(controller.flagSource()); + this.useMultipleEndpoints = Flags.MULTIPLE_GLOBAL_ENDPOINTS.bindTo(controller.flagSource()); this.artifactRepository = artifactRepository; this.applicationStore = applicationStore; this.rotationRepository = new RotationRepository(rotationsConfig, this, curator); this.deploymentTrigger = new DeploymentTrigger(controller, buildService, clock); + this.provisionApplicationCertificate = Flags.PROVISION_APPLICATION_CERTIFICATE.bindTo(controller.flagSource()); + this.applicationCertificateProvider = controller.applicationCertificateProvider(); + // Update serialization format of all applications Once.after(Duration.ofMinutes(1), () -> { Instant start = clock.instant(); @@ -206,7 +223,7 @@ public class ApplicationController { return findGlobalEndpoint(deployment).map(endpoint -> { try { EndpointStatus status = configServer.getGlobalRotationStatus(deployment, endpoint.upstreamName()); - return Collections.singletonMap(endpoint, status); + return Map.of(endpoint, status); } catch (IOException e) { throw new UncheckedIOException("Failed to get rotation status of " + deployment, e); } @@ -226,8 +243,6 @@ public class ApplicationController { * @throws IllegalArgumentException if the application already exists */ public Application createApplication(ApplicationId id, Optional<Credentials> credentials) { - if ( ! (id.instance().isDefault())) // TODO: Support instances properly - throw new IllegalArgumentException("Only the instance name 'default' is supported at the moment"); if (id.instance().isTester()) throw new IllegalArgumentException("'" + id + "' is a tester application!"); try (Lock lock = lock(id)) { @@ -246,7 +261,7 @@ public class ApplicationController { if (credentials.isEmpty()) throw new IllegalArgumentException("Could not create '" + id + "': No credentials provided"); - if (id.instance().isDefault()) // Only store the application permits for non-user applications. + if ( ! id.instance().isTester()) // Only store the application permits for non-user applications. accessControl.createApplication(id, credentials.get()); } LockedApplication application = new LockedApplication(new Application(id, clock.instant()), lock); @@ -281,8 +296,9 @@ public class ApplicationController { Version platformVersion; ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; - Set<String> rotationNames = new HashSet<>(); - Set<String> cnames; + Set<String> legacyRotations = new LinkedHashSet<>(); + Set<ContainerEndpoint> endpoints = new LinkedHashSet<>(); + ApplicationCertificate applicationCertificate; try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(require(applicationId), lock); @@ -320,13 +336,39 @@ public class ApplicationController { // TODO: Remove this when all packages are validated upon submission, as in ApplicationApiHandler.submit(...). verifyApplicationIdentityConfiguration(applicationId.tenant(), applicationPackage, deployingIdentity); + // Assign global rotation - application = withRotation(application, zone); - Application app = application.get(); - // Include global DNS names - cnames = app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).collect(Collectors.toSet()); - // Include rotation ID to ensure that deployment can respond to health checks with rotation ID as Host header - app.rotation().map(RotationId::asString).ifPresent(cnames::add); + if (useMultipleEndpoints.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()).value()) { + application = withRotation(application, zone); + + // Include global DNS names + Application app = application.get(); + app.assignedRotations().stream() + .filter(assignedRotation -> assignedRotation.regions().contains(zone.region())) + .map(assignedRotation -> { + return new ContainerEndpoint( + assignedRotation.clusterId().value(), + Stream.concat( + app.endpointsIn(controller.system(), assignedRotation.endpointId()).legacy(false).asList().stream().map(Endpoint::dnsName), + app.rotations().stream().map(RotationId::asString) + ).collect(Collectors.toList()) + ); + }) + .forEach(endpoints::add); + } else { + application = withRotationLegacy(application, zone); + + // Add both the names we have in DNS for each endpoint as well as name of the rotation so healthchecks works + Application app = application.get(); + app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).forEach(legacyRotations::add); + app.rotations().stream().map(RotationId::asString).forEach(legacyRotations::add); + } + + + // Get application certificate (provisions a new certificate if missing) + application = withApplicationCertificate(application); + applicationCertificate = application.get().applicationCertificate().orElse(null); + // Update application with information from application package if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally() @@ -337,7 +379,7 @@ public class ApplicationController { // Carry out deployment without holding the application lock. options = withVersion(platformVersion, options); - ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames, cnames); + ActivateResult result = deploy(applicationId, applicationPackage, zone, options, legacyRotations, endpoints, applicationCertificate); lockOrThrow(applicationId, application -> store(application.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(), @@ -393,7 +435,7 @@ public class ApplicationController { deploySystemApplicationPackage(application, zone, version); } else { // Deploy by calling node repository directly - application.nodeTypes().forEach(nodeType -> configServer().nodeRepository().upgrade(zone, nodeType, version)); + configServer().nodeRepository().upgrade(zone, application.nodeType(), version); } } @@ -404,7 +446,7 @@ public class ApplicationController { artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); + return deploy(application.id(), applicationPackage, zone, options, Set.of(), Set.of(), /* No application cert */ null); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -412,47 +454,106 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { - return deploy(tester.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); + return deploy(tester.id(), applicationPackage, zone, options, Set.of(), Set.of(), /* No application cert for tester*/ null); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, - Set<String> rotationNames, Set<String> cnames) { + Set<String> legacyRotations, Set<ContainerEndpoint> endpoints, ApplicationCertificate applicationCertificate) { DeploymentId deploymentId = new DeploymentId(application, zone); - ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, cnames, rotationNames, - applicationPackage.zippedContent()); - return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), - applicationPackage.zippedContent().length); + try { + ConfigServer.PreparedApplication preparedApplication = + configServer.deploy(deploymentId, deployOptions, legacyRotations, endpoints, applicationCertificate, applicationPackage.zippedContent()); + return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), + applicationPackage.zippedContent().length); + } finally { + // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that + // any DNS updates can be propagated as early as possible. + routingPolicies.refresh(application, zone); + } } /** Makes sure the application has a global rotation, if eligible. */ - private LockedApplication withRotation(LockedApplication application, ZoneId zone) { + private LockedApplication withRotationLegacy(LockedApplication application, ZoneId zone) { if (zone.environment() == Environment.prod && application.get().deploymentSpec().globalServiceId().isPresent()) { try (RotationLock rotationLock = rotationRepository.lock()) { Rotation rotation = rotationRepository.getOrAssignRotation(application.get(), rotationLock); - application = application.with(rotation.id()); + application = application.with(createDefaultGlobalIdRotation(application.get(), rotation)); store(application); // store assigned rotation even if deployment fails - boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) - .value(); - EndpointList globalEndpoints = application.get() - .endpointsIn(controller.system()) - .scope(Endpoint.Scope.global); + .endpointsIn(controller.system()) + .scope(Endpoint.Scope.global); + globalEndpoints.main().ifPresent(mainEndpoint -> { registerCname(mainEndpoint.dnsName(), rotation.name()); - if (redirectLegacyDns) { - globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), mainEndpoint.dnsName())); - } else { - globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), rotation.name())); - } + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), rotation.name())); }); } } return application; } + private List<AssignedRotation> createDefaultGlobalIdRotation(Application application, Rotation rotation) { + // This is guaranteed by .withRotationLegacy, but add this to make inspections accept the use of .get() below + assert application.deploymentSpec().globalServiceId().isPresent(); + + final Set<RegionName> regions = application.deploymentSpec().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + + final var assignment = new AssignedRotation( + ClusterSpec.Id.from(application.deploymentSpec().globalServiceId().get()), + EndpointId.default_(), + rotation.id(), + regions + ); + + return List.of(assignment); + } + + /** Makes sure the application has a global rotation, if eligible. */ + private LockedApplication withRotation(LockedApplication application, ZoneId zone) { + if (zone.environment() == Environment.prod) { + try (RotationLock rotationLock = rotationRepository.lock()) { + final var rotations = rotationRepository.getOrAssignRotations(application.get(), rotationLock); + application = application.with(rotations); + store(application); // store assigned rotation even if deployment fails + registerAssignedRotationCnames(application.get()); + } + } + return application; + } + + private void registerAssignedRotationCnames(Application application) { + application.assignedRotations().forEach(assignedRotation -> { + final var endpoints = application + .endpointsIn(controller.system(), assignedRotation.endpointId()) + .scope(Endpoint.Scope.global); + + final var maybeRotation = rotationRepository.getRotation(assignedRotation.rotationId()); + + maybeRotation.ifPresent(rotation -> { + endpoints.main().ifPresent(mainEndpoint -> { + registerCname(mainEndpoint.dnsName(), rotation.name()); + }); + }); + }); + } + + private LockedApplication withApplicationCertificate(LockedApplication application) { + ApplicationId applicationId = application.get().id(); + + // TODO: Verify that the application is deploying to a zone where certificate provisioning is enabled + boolean provisionCertificate = provisionApplicationCertificate.with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + if (provisionCertificate) { + application = application.withApplicationCertificate( + Optional.of(applicationCertificateProvider.requestCaSignedCertificate(applicationId))); + } + return application; + } + private ActivateResult unexpectedDeployment(ApplicationId application, ZoneId zone) { Log logEntry = new Log(); logEntry.level = "WARNING"; @@ -460,8 +561,8 @@ public class ApplicationController { logEntry.message = "Ignoring deployment of application '" + application + "' to " + zone + " as a deployment is not currently expected"; PrepareResponse prepareResponse = new PrepareResponse(); - prepareResponse.log = Collections.singletonList(logEntry); - prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), Collections.emptyList()); + prepareResponse.log = List.of(logEntry); + prepareResponse.configChangeActions = new ConfigChangeActions(List.of(), List.of()); return new ActivateResult(new RevisionId("0"), prepareResponse, 0); } @@ -512,24 +613,57 @@ public class ApplicationController { controller.nameServiceForwarder().createCname(RecordName.from(name), RecordData.fqdn(targetName), Priority.normal); } - /** Returns the endpoints of the deployment, or an empty list if the request fails */ - public Optional<List<URI>> getDeploymentEndpoints(DeploymentId deploymentId) { + /** Returns the endpoints of the deployment, or empty if the request fails */ + public List<URI> getDeploymentEndpoints(DeploymentId deploymentId) { if ( ! get(deploymentId.applicationId()) .map(application -> application.deployments().containsKey(deploymentId.zoneId())) .orElse(deploymentId.applicationId().instance().isTester())) throw new NotExistsException("Deployment", deploymentId.toString()); try { - return Optional.of(ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream() - .map(RoutingEndpoint::endpoint) - .map(URI::create) - .iterator())); + return ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream() + .map(RoutingEndpoint::endpoint) + .map(URI::create) + .iterator()); } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId + ": " + Exceptions.toMessageString(e)); - return Optional.empty(); + return Collections.emptyList(); + } + } + + /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */ + public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId id) { + if ( ! get(id.applicationId()) + .map(application -> application.deployments().containsKey(id.zoneId())) + .orElse(id.applicationId().instance().isTester())) + throw new NotExistsException("Deployment", id.toString()); + + // TODO jvenstad: Swap to use routingPolicies first, when this is ready. + try { + var endpoints = routingGenerator.clusterEndpoints(id); + if ( ! endpoints.isEmpty()) + return endpoints; + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": " + Exceptions.toMessageString(e)); } + return routingPolicies.get(id).stream() + .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) + .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), + policy -> policy.endpointIn(controller.system()).url())); + } + + /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */ + public Map<ZoneId, Map<ClusterSpec.Id, URI>> clusterEndpoints(ApplicationId id, Collection<ZoneId> zones) { + Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments = new TreeMap<>(Comparator.comparing(ZoneId::value)); + for (ZoneId zone : zones) { + var endpoints = clusterEndpoints(new DeploymentId(id, zone)); + if ( ! endpoints.isEmpty()) + deployments.put(zone, endpoints); + } + return Collections.unmodifiableMap(deployments); } /** @@ -556,12 +690,23 @@ public class ApplicationController { // TODO: Make this one transaction when database is moved to ZooKeeper instances.forEach(id -> lockOrThrow(id, application -> { if ( ! application.get().deployments().isEmpty()) - throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments"); + throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments in: " + + application.get().deployments().keySet().stream().map(ZoneId::toString) + .sorted().collect(Collectors.joining(", "))); curator.removeApplication(id); applicationStore.removeAll(id); applicationStore.removeAll(TesterId.of(id)); + application.get().assignedRotations().forEach(assignedRotation -> { + final var endpoints = application.get().endpointsIn(controller.system(), assignedRotation.endpointId()); + endpoints.asList().stream() + .map(Endpoint::dnsName) + .forEach(name -> { + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal); + }); + }); + log.info("Deleted " + application); })); @@ -641,9 +786,10 @@ public class ApplicationController { private LockedApplication deactivate(LockedApplication application, ZoneId zone) { try { configServer.deactivate(new DeploymentId(application.get().id(), zone)); - } - catch (NotFoundException ignored) { + } catch (NotFoundException ignored) { // ok; already gone + } finally { + routingPolicies.refresh(application.get().id(), zone); } return application.withoutDeploymentIn(zone); } @@ -703,9 +849,8 @@ public class ApplicationController { return rotationRepository; } - /** Returns all known routing policies for given application */ - public Set<RoutingPolicy> routingPolicies(ApplicationId application) { - return curator.readRoutingPolicies(application); + public RoutingPolicies routingPolicies() { + return routingPolicies; } /** Sort given list of applications by application ID */ @@ -767,7 +912,7 @@ public class ApplicationController { if (!"warn".equalsIgnoreCase(log.level) && !"warning".equalsIgnoreCase(log.level)) continue; warnings.merge(DeploymentMetrics.Warning.all, 1, Integer::sum); } - return Collections.unmodifiableMap(warnings); + return Map.copyOf(warnings); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index fde34e62ae4..08c95d1ecab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -8,18 +8,19 @@ import com.yahoo.component.Vtag; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHub; +import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; @@ -77,11 +78,12 @@ public class Controller extends AbstractComponent { private final ZoneRegistry zoneRegistry; private final ConfigServer configServer; private final MetricsService metricsService; - private final Chef chef; private final Mailer mailer; private final AuditLogger auditLogger; private final FlagSource flagSource; private final NameServiceForwarder nameServiceForwarder; + private final ApplicationCertificateProvider applicationCertificateProvider; + private final MavenRepository mavenRepository; /** * Creates a controller @@ -91,24 +93,26 @@ public class Controller extends AbstractComponent { @Inject public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, ZoneRegistry zoneRegistry, ConfigServer configServer, MetricsService metricsService, - RoutingGenerator routingGenerator, Chef chef, + RoutingGenerator routingGenerator, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, - BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource) { + BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource, + MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider) { this(curator, rotationsConfig, gitHub, zoneRegistry, - configServer, metricsService, routingGenerator, chef, + configServer, metricsService, routingGenerator, Clock.systemUTC(), accessControl, artifactRepository, applicationStore, testerCloud, - buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource); + buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource, + mavenRepository, applicationCertificateProvider); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, ZoneRegistry zoneRegistry, ConfigServer configServer, MetricsService metricsService, - RoutingGenerator routingGenerator, Chef chef, Clock clock, + RoutingGenerator routingGenerator, Clock clock, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Supplier<String> hostnameSupplier, - Mailer mailer, FlagSource flagSource) { + Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); @@ -116,11 +120,12 @@ public class Controller extends AbstractComponent { this.zoneRegistry = Objects.requireNonNull(zoneRegistry, "ZoneRegistry cannot be null"); this.configServer = Objects.requireNonNull(configServer, "ConfigServer cannot be null"); this.metricsService = Objects.requireNonNull(metricsService, "MetricsService cannot be null"); - this.chef = Objects.requireNonNull(chef, "Chef cannot be null"); this.clock = Objects.requireNonNull(clock, "Clock cannot be null"); this.mailer = Objects.requireNonNull(mailer, "Mailer cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.nameServiceForwarder = new NameServiceForwarder(curator); + this.applicationCertificateProvider = Objects.requireNonNull(applicationCertificateProvider); + this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); jobController = new JobController(this, runDataStore, Objects.requireNonNull(testerCloud)); applicationController = new ApplicationController(this, curator, accessControl, @@ -163,9 +168,9 @@ public class Controller extends AbstractComponent { public ZoneRegistry zoneRegistry() { return zoneRegistry; } - public NameServiceForwarder nameServiceForwarder() { - return nameServiceForwarder; - } + public NameServiceForwarder nameServiceForwarder() { return nameServiceForwarder; } + + public MavenRepository mavenRepository() { return mavenRepository; } public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) { @@ -286,10 +291,6 @@ public class Controller extends AbstractComponent { return zoneRegistry.system(); } - public Chef chefClient() { - return chef; - } - public CuratorDb curator() { return curator; } @@ -298,6 +299,10 @@ public class Controller extends AbstractComponent { return auditLogger; } + public ApplicationCertificateProvider applicationCertificateProvider() { + return applicationCertificateProvider; + } + /** Returns all other roles the given tenant role implies. */ public Set<Role> impliedRoles(TenantRole role) { return Stream.concat(Roles.tenantRoles(role.tenant()).stream(), @@ -315,8 +320,8 @@ public class Controller extends AbstractComponent { } private Set<CloudName> clouds() { - return zoneRegistry.zones().all().ids().stream() - .map(ZoneId::cloud) + return zoneRegistry.zones().all().zones().stream() + .map(ZoneApi::getCloudName) .collect(Collectors.toUnmodifiableSet()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index d1a5360625c..1e032ab25ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -11,11 +11,13 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -57,8 +59,9 @@ public class LockedApplication { private final OptionalInt majorVersion; private final ApplicationMetrics metrics; private final Optional<String> pemDeployKey; - private final Optional<RotationId> rotation; + private final List<AssignedRotation> rotations; private final Map<HostName, RotationStatus> rotationStatus; + private final Optional<ApplicationCertificate> applicationCertificate; /** * Used to create a locked application @@ -72,7 +75,7 @@ public class LockedApplication { application.deployments(), application.deploymentJobs(), application.change(), application.outstandingChange(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), - application.pemDeployKey(), application.rotation(), application.rotationStatus()); + application.pemDeployKey(), application.assignedRotations(), application.rotationStatus(), application.applicationCertificate()); } private LockedApplication(Lock lock, ApplicationId id, Instant createdAt, @@ -80,7 +83,7 @@ public class LockedApplication { Map<ZoneId, Deployment> deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional<IssueId> ownershipIssueId, Optional<User> owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional<String> pemDeployKey, - Optional<RotationId> rotation, Map<HostName, RotationStatus> rotationStatus) { + List<AssignedRotation> rotations, Map<HostName, RotationStatus> rotationStatus, Optional<ApplicationCertificate> applicationCertificate) { this.lock = lock; this.id = id; this.createdAt = createdAt; @@ -95,43 +98,44 @@ public class LockedApplication { this.majorVersion = majorVersion; this.metrics = metrics; this.pemDeployKey = pemDeployKey; - this.rotation = rotation; + this.rotations = rotations; this.rotationStatus = rotationStatus; + this.applicationCertificate = applicationCertificate; } /** Returns a read-only copy of this */ public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withBuiltInternally(boolean builtInternally) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.with(issueId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withJobPause(JobType jobType, OptionalLong pausedUntil) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withPause(jobType, pausedUntil), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withJobCompletion(long projectId, JobType jobType, JobStatus.JobRun completion, @@ -139,14 +143,14 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withCompletion(projectId, jobType, completion, jobError), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, rotation, rotationStatus); + pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withJobTriggering(JobType jobType, JobStatus.JobRun job) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withTriggering(jobType, job), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, @@ -198,45 +202,45 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.without(jobType), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withChange(Change change) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withOutstandingChange(Change outstandingChange) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, Optional.ofNullable(issueId), owner, - majorVersion, metrics, pemDeployKey, rotation, rotationStatus); + majorVersion, metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, Optional.ofNullable(owner), majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus, applicationCertificate); } /** Set a major version for this, or set to null to remove any major version override */ @@ -244,31 +248,31 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication with(MetricsService.ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withPemDeployKey(String pemDeployKey) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, Optional.ofNullable(pemDeployKey), rotation, rotationStatus); + metrics, Optional.ofNullable(pemDeployKey), rotations, rotationStatus, applicationCertificate); } - public LockedApplication with(RotationId rotation) { + public LockedApplication with(List<AssignedRotation> assignedRotations) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, Optional.of(rotation), rotationStatus); + metrics, pemDeployKey, assignedRotations, rotationStatus, applicationCertificate); } public LockedApplication withRotationStatus(Map<HostName, RotationStatus> rotationStatus) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication with(ZoneId zoneId, List<ClusterMetrics> clusterMetrics) { @@ -277,6 +281,12 @@ public class LockedApplication { return with(deployment.withClusterMetrics(clusterMetrics)); } + public LockedApplication withApplicationCertificate(Optional<ApplicationCertificate> applicationCertificate) { + return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, + deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); + } + /** Don't expose non-leaf sub-objects. */ private LockedApplication with(Deployment deployment) { Map<ZoneId, Deployment> deployments = new LinkedHashMap<>(this.deployments); @@ -287,7 +297,7 @@ public class LockedApplication { private LockedApplication with(Map<ZoneId, Deployment> deployments) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java new file mode 100644 index 00000000000..ec13066d069 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -0,0 +1,80 @@ +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RegionName; +import com.yahoo.vespa.hosted.controller.rotation.RotationId; + +import java.util.Collection; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Contains the tuple of [clusterId, endpointId, rotationId, regions[]], to keep track + * of which services have assigned which rotations under which name. + * + * @author ogronnesby + */ +public class AssignedRotation { + private final ClusterSpec.Id clusterId; + private final EndpointId endpointId; + private final RotationId rotationId; + private final Set<RegionName> regions; + + public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId, Set<RegionName> regions) { + this.clusterId = requireNonEmpty(clusterId, clusterId.value(), "clusterId"); + this.endpointId = Objects.requireNonNull(endpointId); + this.rotationId = Objects.requireNonNull(rotationId); + this.regions = Set.copyOf(Objects.requireNonNull(regions)); + } + + public ClusterSpec.Id clusterId() { return clusterId; } + public EndpointId endpointId() { return endpointId; } + public RotationId rotationId() { return rotationId; } + public Set<RegionName> regions() { return regions; } + + @Override + public String toString() { + return "AssignedRotation{" + + "clusterId=" + clusterId + + ", endpointId='" + endpointId + '\'' + + ", rotationId=" + rotationId + + ", regions=" + regions + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AssignedRotation that = (AssignedRotation) o; + return clusterId.equals(that.clusterId) && + endpointId.equals(that.endpointId) && + rotationId.equals(that.rotationId) && + regions.equals(that.regions); + } + + @Override + public int hashCode() { + return Objects.hash(clusterId, endpointId, rotationId, regions); + } + + private static <T> T requireNonEmpty(T object, String value, String field) { + Objects.requireNonNull(object); + Objects.requireNonNull(value); + if (value.isEmpty()) { + throw new IllegalArgumentException("Field '" + field + "' was empty"); + } + return object; + } + + /** Convenience method intended for tests */ + public static AssignedRotation fromStrings(String clusterId, String endpointId, String rotationId, Collection<String> regions) { + return new AssignedRotation( + new ClusterSpec.Id(clusterId), + new EndpointId(endpointId), + new RotationId(rotationId), + regions.stream().map(RegionName::from).collect(Collectors.toSet()) + ); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index ce7af03aa7e..5dccd5c8120 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -21,6 +21,7 @@ public class Endpoint { public static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com"; public static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; public static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; + public static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud"; private final URI url; private final Scope scope; @@ -118,7 +119,7 @@ public class Endpoint { private static String separator(SystemName system, boolean directRouting, boolean tls) { if (!tls) return "."; if (directRouting) return "."; - if (isPublic(system)) return "."; + if (system.isPublic()) return "."; return "--"; } @@ -140,8 +141,8 @@ public class Endpoint { } private static String systemPart(SystemName system, String separator) { - if (system == SystemName.main || isPublic(system)) return ""; - return system.name() + separator; + if (!system.isCd()) return ""; + return system.value() + separator; } private static String dnsSuffix(SystemName system, boolean legacy) { @@ -151,16 +152,13 @@ public class Endpoint { if (legacy) return YAHOO_DNS_SUFFIX; return OATH_DNS_SUFFIX; case Public: - case vaas: return PUBLIC_DNS_SUFFIX; + case PublicCd: + return PUBLIC_CD_DNS_SUFFIX; default: throw new IllegalArgumentException("No DNS suffix declared for system " + system); } } - private static boolean isPublic(SystemName system) { // TODO: Remove and inline once we're down to one - return system == SystemName.Public || system == SystemName.vaas; - } - /** An endpoint's scope */ public enum Scope { @@ -219,6 +217,7 @@ public class Endpoint { private ZoneId zone; private ClusterSpec.Id cluster; private RotationName rotation; + private EndpointId endpointId; private Port port; private boolean legacy = false; private boolean directRouting = false; @@ -229,8 +228,8 @@ public class Endpoint { /** Sets the cluster and zone target of this */ public EndpointBuilder target(ClusterSpec.Id cluster, ZoneId zone) { - if (rotation != null) { - throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + if (rotation != null || endpointId != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); } this.cluster = cluster; this.zone = zone; @@ -239,13 +238,22 @@ public class Endpoint { /** Sets the rotation target of this */ public EndpointBuilder target(RotationName rotation) { - if (cluster != null && zone != null) { - throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + if ((cluster != null && zone != null) || endpointId != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); } this.rotation = rotation; return this; } + /** Sets the endpoint ID as defines in deployments.xml */ + public EndpointBuilder named(EndpointId endpointId) { + if (rotation != null || cluster != null || zone != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); + } + this.endpointId = endpointId; + return this; + } + /** Sets the port of this */ public EndpointBuilder on(Port port) { this.port = port; @@ -271,10 +279,12 @@ public class Endpoint { name = cluster.value(); } else if (rotation != null) { name = rotation.value(); + } else if (endpointId != null) { + name = endpointId.id(); } else { throw new IllegalArgumentException("Must set either cluster or rotation target"); } - if (isPublic(system) && !directRouting) { + if (system.isPublic() && !directRouting) { throw new IllegalArgumentException("Public system only supports direct routing endpoints"); } if (directRouting && !port.isDefault()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java new file mode 100644 index 00000000000..13c242c7b5f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java @@ -0,0 +1,53 @@ +package com.yahoo.vespa.hosted.controller.application; + +import java.util.Objects; + +/** + * A type to represent the ID of an endpoint. This is typically the first part of + * an endpoint name. + * + * @author ogronnesby + */ +public class EndpointId { + private static final EndpointId DEFAULT = new EndpointId("default"); + + private final String id; + + public EndpointId(String id) { + this.id = requireNotEmpty(id); + } + + public String id() { return id; } + + @Override + public String toString() { + return "EndpointId{" + + "id='" + id + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EndpointId that = (EndpointId) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + private static String requireNotEmpty(String input) { + Objects.requireNonNull(input); + if (input.isEmpty()) { + throw new IllegalArgumentException("The value EndpointId was empty"); + } + return input; + } + + public static EndpointId default_() { return DEFAULT; } + + public static EndpointId of(String id) { return new EndpointId(id); } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index 0c04a1f099c..d9aea783880 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -25,13 +25,6 @@ public class EndpointList { private final List<Endpoint> endpoints; private EndpointList(List<Endpoint> endpoints) { - long mainEndpoints = endpoints.stream() - .filter(endpoint -> endpoint.scope() == Endpoint.Scope.global) - .filter(Predicate.not(Endpoint::directRouting)) - .filter(Predicate.not(Endpoint::legacy)).count(); - if (mainEndpoints > 1) { - throw new IllegalArgumentException("Can have only 1 non-legacy global endpoint, got " + endpoints); - } if (endpoints.stream().distinct().count() != endpoints.size()) { throw new IllegalArgumentException("Expected all endpoints to be distinct, got " + endpoints); } @@ -67,16 +60,14 @@ public class EndpointList { } /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */ - public static EndpointList defaultGlobal(ApplicationId application, SystemName system) { - // Rotation name is always default in the routing layer - RotationName rotation = RotationName.from("default"); + public static EndpointList create(ApplicationId application, EndpointId endpointId, SystemName system) { switch (system) { case cd: case main: return new EndpointList(List.of( - Endpoint.of(application).target(rotation).on(Port.plain(4080)).legacy().in(system), - Endpoint.of(application).target(rotation).on(Port.tls(4443)).legacy().in(system), - Endpoint.of(application).target(rotation).on(Port.tls(4443)).in(system) + Endpoint.of(application).named(endpointId).on(Port.plain(4080)).legacy().in(system), + Endpoint.of(application).named(endpointId).on(Port.tls(4443)).legacy().in(system), + Endpoint.of(application).named(endpointId).on(Port.tls(4443)).in(system) )); } return EMPTY; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 708133d2a07..0d6da51c492 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; @@ -9,8 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; +import java.util.Optional; /** * This represents a system-level application in hosted Vespa. All infrastructure nodes in a hosted Vespa zones are @@ -21,25 +21,18 @@ import java.util.stream.Collectors; public enum SystemApplication { configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), + configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), - configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config, configServerHost), - zone(ApplicationId.from("hosted-vespa", "routing", "default"), Set.of(NodeType.proxy, NodeType.host), - configServerHost, proxyHost, configServer); + proxy(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, proxyHost, configServer), + tenantHost(ApplicationId.from("hosted-vespa", "tenant-host", "default"), NodeType.host); private final ApplicationId id; - private final Set<NodeType> nodeTypes; + private final NodeType nodeType; private final List<SystemApplication> dependencies; SystemApplication(ApplicationId id, NodeType nodeType, SystemApplication... dependencies) { - this(id, Set.of(nodeType), dependencies); - } - - SystemApplication(ApplicationId id, Set<NodeType> nodeTypes, SystemApplication... dependencies) { - if (nodeTypes.isEmpty()) { - throw new IllegalArgumentException("Node types must be non-empty"); - } this.id = id; - this.nodeTypes = Set.copyOf(nodeTypes); + this.nodeType = nodeType; this.dependencies = List.of(dependencies); } @@ -47,9 +40,9 @@ public enum SystemApplication { return id; } - /** The node type(s) that are implicitly allocated to this */ - public Set<NodeType> nodeTypes() { - return nodeTypes; + /** The node type that is implicitly allocated to this */ + public NodeType nodeType() { + return nodeType; } /** Returns the system applications that should upgrade before this */ @@ -57,22 +50,22 @@ public enum SystemApplication { /** Returns whether this system application has an application package */ public boolean hasApplicationPackage() { - return this == zone; + return this == proxy; } /** Returns whether config for this application has converged in given zone */ - public boolean configConvergedIn(ZoneId zone, Controller controller) { + public boolean configConvergedIn(ZoneId zone, Controller controller, Optional<Version> version) { if (!hasApplicationPackage()) { return true; } - return controller.configServer().serviceConvergence(new DeploymentId(id(), zone)) + return controller.configServer().serviceConvergence(new DeploymentId(id(), zone), version) .map(ServiceConvergence::converged) .orElse(false); } /** Returns the node types of this that should receive OS upgrades */ - public Set<NodeType> nodeTypesWithUpgradableOs() { - return nodeTypes().stream().filter(NodeType::isDockerHost).collect(Collectors.toSet()); + public boolean isEligibleForOsUpgrades() { + return nodeType.isDockerHost(); } /** All known system applications */ @@ -82,7 +75,7 @@ public enum SystemApplication { @Override public String toString() { - return String.format("system application %s of type %s", id, nodeTypes); + return String.format("system application %s of type %s", id, nodeType); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java index c467a4a0acd..aefe8ae7b48 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java @@ -109,6 +109,7 @@ public class AuditLog { public enum Method { POST, PATCH, + PUT, DELETE } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index a8130d60cc5..b4fe8ea2971 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -92,7 +92,7 @@ public class DeploymentTrigger { * trigger next. */ public void notifyOfCompletion(JobReport report) { - log.log(LogLevel.INFO, String.format("Notified of %s for %s of %s (%d)", + log.log(LogLevel.DEBUG, String.format("Notified of %s for %s of %s (%d)", report.jobError().map(e -> e.toString() + " error") .orElse("success"), report.jobType(), @@ -124,14 +124,16 @@ public class DeploymentTrigger { } } else { - triggering = application.get().deploymentJobs().statusOf(report.jobType()) - .filter(job -> job.lastTriggered().isPresent() - && job.lastCompleted() - .map(completion -> ! completion.at().isAfter(job.lastTriggered().get().at())) - .orElse(true)) - .orElseThrow(() -> new IllegalStateException("Notified of completion of " + report.jobType().jobName() + " for " + - report.applicationId() + ", but that has neither been triggered nor deployed")) - .lastTriggered().get(); + Optional<JobStatus> status = application.get().deploymentJobs().statusOf(report.jobType()); + triggering = status.filter(job -> job.lastTriggered().isPresent() + && job.lastCompleted() + .map(completion -> ! completion.at().isAfter(job.lastTriggered().get().at())) + .orElse(true)) + .orElseThrow(() -> new IllegalStateException("Notified of completion of " + report.jobType().jobName() + " for " + + report.applicationId() + ", but that has not been triggered; last was " + + status.flatMap(job -> job.lastTriggered().map(run -> run.at().toString())) + .orElse("never"))) + .lastTriggered().get(); } application = application.withJobCompletion(report.projectId(), report.jobType(), @@ -181,7 +183,7 @@ public class DeploymentTrigger { * the project id is removed from the application owning the job, to prevent further trigger attempts. */ public boolean trigger(Job job) { - log.log(LogLevel.INFO, String.format("Triggering %s: %s", job, job.triggering)); + log.log(LogLevel.DEBUG, String.format("Triggering %s: %s", job, job.triggering)); try { applications().lockOrThrow(job.applicationId(), application -> { if (application.get().deploymentJobs().deployedInternally()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 472a0c5fb7e..ce0e7c0dbab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -11,13 +11,11 @@ import com.yahoo.config.application.api.Notifications.When; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.ActivateResult; @@ -28,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -41,7 +38,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.PrintStream; import java.io.UncheckedIOException; import java.net.URI; @@ -57,7 +53,6 @@ import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.yahoo.config.application.api.Notifications.Role.author; import static com.yahoo.config.application.api.Notifications.When.failing; @@ -66,6 +61,7 @@ import static com.yahoo.log.LogLevel.DEBUG; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.BAD_REQUEST; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.CERTIFICATE_NOT_READY; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.PARENT_HOST_NOT_READY; @@ -99,10 +95,12 @@ public class InternalStepRunner implements StepRunner { static final Duration installationTimeout = Duration.ofMinutes(150); private final Controller controller; + private final TestConfigSerializer testConfigSerializer; private final DeploymentFailureMails mails; public InternalStepRunner(Controller controller) { this.controller = controller; + this.testConfigSerializer = new TestConfigSerializer(controller.system()); this.mails = new DeploymentFailureMails(controller.zoneRegistry()); } @@ -234,7 +232,8 @@ public class InternalStepRunner implements StepRunner { if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() || e.getErrorCode() == ACTIVATION_CONFLICT || e.getErrorCode() == APPLICATION_LOCK_FAILURE - || e.getErrorCode() == PARENT_HOST_NOT_READY) { + || e.getErrorCode() == PARENT_HOST_NOT_READY + || e.getErrorCode() == CERTIFICATE_NOT_READY) { logger.log("Will retry, because of '" + e.getErrorCode() + "' deploying:\n" + e.getMessage()); return Optional.empty(); } @@ -268,9 +267,15 @@ public class InternalStepRunner implements StepRunner { logger.log("Checking installation of " + platform + " and " + application.id() + " ..."); if ( nodesConverged(id.application(), id.type(), platform, logger) - && servicesConverged(id.application(), id.type(), logger)) { - logger.log("Installation succeeded!"); - return Optional.of(running); + && servicesConverged(id.application(), id.type(), platform, logger)) { + if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { + logger.log("Installation succeeded!"); + return Optional.of(running); + } + else if (timedOut(deployment.get(), endpointTimeout)) { + logger.log(WARNING, "Endpoints failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); + return Optional.of(error); + } } if (timedOut(deployment.get(), installationTimeout)) { @@ -292,9 +297,15 @@ public class InternalStepRunner implements StepRunner { Version platform = controller.jobController().run(id).get().versions().targetPlatform(); logger.log("Checking installation of tester container ..."); if ( nodesConverged(id.tester().id(), id.type(), platform, logger) - && servicesConverged(id.tester().id(), id.type(), logger)) { - logger.log("Tester container successfully installed!"); - return Optional.of(running); + && servicesConverged(id.tester().id(), id.type(), platform, logger)) { + if (endpointsAvailable(id.tester().id(), id.type().zone(controller.system()), logger)) { + logger.log("Tester container successfully installed!"); + return Optional.of(running); + } + else if (timedOut(deployment.get(), endpointTimeout)) { + logger.log(WARNING, "Tester failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); + return Optional.of(error); + } } if (timedOut(deployment.get(), installationTimeout)) { @@ -306,6 +317,27 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } + private boolean endpointsAvailable(ApplicationId id, ZoneId zoneId, DualLogger logger) { + logger.log("Attempting to find deployment endpoints ..."); + var endpoints = controller.applications().clusterEndpoints(id, Set.of(zoneId)); + if ( ! endpoints.containsKey(zoneId)) { + logger.log("Endpoints not yet ready."); + return false; + } + logEndpoints(endpoints, logger); + return true; + } + + private void logEndpoints(Map<ZoneId, Map<ClusterSpec.Id, URI>> endpoints, DualLogger logger) { + List<String> messages = new ArrayList<>(); + messages.add("Found endpoints:"); + endpoints.forEach((zone, uris) -> { + messages.add("- " + zone); + uris.forEach((cluster, uri) -> messages.add(" |-- " + uri + " (" + cluster + ")")); + }); + logger.log(messages); + } + private boolean nodesConverged(ApplicationId id, JobType type, Version target, DualLogger logger) { List<Node> nodes = controller.configServer().nodeRepository().list(type.zone(controller.system()), id, ImmutableSet.of(active, reserved)); List<String> statuses = nodes.stream() @@ -325,9 +357,10 @@ public class InternalStepRunner implements StepRunner { && node.rebootGeneration() >= node.wantedRebootGeneration()); } - private boolean servicesConverged(ApplicationId id, JobType type, DualLogger logger) { - Optional<ServiceConvergence> convergence = controller.configServer().serviceConvergence(new DeploymentId(id, type.zone(controller.system()))); - if ( ! convergence.isPresent()) { + private boolean servicesConverged(ApplicationId id, JobType type, Version platform, DualLogger logger) { + var convergence = controller.configServer().serviceConvergence(new DeploymentId(id, type.zone(controller.system())), + Optional.of(platform)); + if (convergence.isEmpty()) { logger.log("Config status not currently available -- will retry."); return false; } @@ -341,6 +374,8 @@ public class InternalStepRunner implements StepRunner { serviceStatus.currentGeneration() == -1 ? "not started!" : Long.toString(serviceStatus.currentGeneration()))) .collect(Collectors.toList()); logger.log(statuses); + if (statuses.isEmpty()) + logger.log("All services on wanted config generation."); return convergence.get().converged(); } @@ -352,45 +387,34 @@ public class InternalStepRunner implements StepRunner { return Optional.of(aborted); } - Set<ZoneId> zones = testedZoneAndProductionZones(id); + Set<ZoneId> zones = controller.jobController().testedZoneAndProductionZones(id.application(), id.type()); logger.log("Attempting to find endpoints ..."); - Map<ZoneId, List<URI>> endpoints = deploymentEndpoints(id.application(), zones); - List<String> messages = new ArrayList<>(); - messages.add("Found endpoints"); - endpoints.forEach((zone, uris) -> { - messages.add("- " + zone); - uris.forEach(uri -> messages.add(" |-- " + uri)); - }); - logger.log(messages); - if ( ! endpoints.containsKey(id.type().zone(controller.system()))) { - if (timedOut(deployment.get(), endpointTimeout)) { - logger.log(WARNING, "Endpoints failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); - return Optional.of(error); - } - - logger.log("Endpoints for the deployment to test are not yet ready."); - return Optional.empty(); + var endpoints = controller.applications().clusterEndpoints(id.application(), zones); + if ( ! endpoints.containsKey(id.type().zone(controller.system())) && timedOut(deployment.get(), endpointTimeout)) { + logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); + return Optional.of(error); } - - Map<ZoneId, List<String>> clusters = listClusters(id.application(), zones); + logEndpoints(endpoints, logger); Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); - if (testerEndpoint.isPresent() && controller.jobController().cloud().ready(testerEndpoint.get())) { + if (testerEndpoint.isEmpty() && timedOut(deployment.get(), endpointTimeout)) { + logger.log(WARNING, "Endpoints for the tester container vanished again, while it was still active!"); + return Optional.of(error); + } + + if (controller.jobController().cloud().ready(testerEndpoint.get())) { logger.log("Starting tests ..."); controller.jobController().cloud().startTests(testerEndpoint.get(), TesterCloud.Suite.of(id.type()), - testConfig(id.application(), id.type().zone(controller.system()), - controller.system(), endpoints, clusters)); + testConfigSerializer.configJson(id.application(), + id.type(), + endpoints, + listClusters(id.application(), zones))); return Optional.of(running); } - if (timedOut(deployment.get(), endpointTimeout)) { - logger.log(WARNING, "Endpoint for tester failed to show up within " + endpointTimeout.toMinutes() + " minutes of real deployment!"); - return Optional.of(error); - } - - logger.log("Endpoints of tester container not yet available."); + logger.log("Tester container not yet ready."); return Optional.empty(); } @@ -430,7 +454,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> copyVespaLogs(RunId id, DualLogger logger) { ZoneId zone = id.type().zone(controller.system()); - if (controller.applications().require(id.application()).deployments().containsKey(zone)) + if (deployment(id.application(), id.type()).isPresent()) try { logger.log("Copying Vespa log from nodes of " + id.application() + " in " + zone + " ..."); List<LogEntry> entries = new ArrayList<>(); @@ -451,20 +475,33 @@ public class InternalStepRunner implements StepRunner { } catch (Exception e) { logger.log(INFO, "Failure getting vespa logs for " + id, e); + return Optional.of(error); } - return Optional.of(running); // Don't let failure here stop cleanup. + return Optional.of(running); } private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.applications().deactivate(id.application(), id.type().zone(controller.system())); - return Optional.of(running); + try { + logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.applications().deactivate(id.application(), id.type().zone(controller.system())); + return Optional.of(running); + } + catch (RuntimeException e) { + logger.log(WARNING, "Failed deleting application " + id.application(), e); + return Optional.of(error); + } } private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.jobController().deactivateTester(id.tester(), id.type()); - return Optional.of(running); + try { + logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.jobController().deactivateTester(id.tester(), id.type()); + return Optional.of(running); + } + catch (RuntimeException e) { + logger.log(WARNING, "Failed deleting tester of " + id.application(), e); + return Optional.of(error); + } } private Optional<RunStatus> report(RunId id, DualLogger logger) { @@ -481,7 +518,8 @@ public class InternalStepRunner implements StepRunner { }); } catch (IllegalStateException e) { - logger.log(INFO, "Job '" + id.type() + "'no longer supposed to run?:", e); + logger.log(INFO, "Job '" + id.type() + "' no longer supposed to run?", e); + return Optional.of(error); } return Optional.of(running); } @@ -526,6 +564,7 @@ public class InternalStepRunner implements StepRunner { /** Returns the real application with the given id. */ private Application application(ApplicationId id) { + controller.applications().lockOrThrow(id, __ -> { }); // Memory fence. return controller.applications().require(id); } @@ -571,23 +610,6 @@ public class InternalStepRunner implements StepRunner { throw new IllegalStateException("No step deploys to the zone this run is for!"); } - /** Returns a stream containing the zone of the deployment tested in the given run, and all production zones for the application. */ - private Set<ZoneId> testedZoneAndProductionZones(RunId id) { - return Stream.concat(Stream.of(id.type().zone(controller.system())), - application(id.application()).productionDeployments().keySet().stream()) - .collect(Collectors.toSet()); - } - - /** Returns all endpoints for all current deployments of the given real application. */ - private Map<ZoneId, List<URI>> deploymentEndpoints(ApplicationId id, Iterable<ZoneId> zones) { - ImmutableMap.Builder<ZoneId, List<URI>> deployments = ImmutableMap.builder(); - for (ZoneId zone : zones) - controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) - .filter(endpoints -> ! endpoints.isEmpty()) - .ifPresent(endpoints -> deployments.put(zone, endpoints)); - return deployments.build(); - } - /** Returns all content clusters in all current deployments of the given real application. */ private Map<ZoneId, List<String>> listClusters(ApplicationId id, Iterable<ZoneId> zones) { ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder(); @@ -603,12 +625,12 @@ public class InternalStepRunner implements StepRunner { String flavor = testerFlavor.orElse("d-1-4-50"); int memoryGb = Integer.parseInt(flavor.split("-")[2]); // Memory available in tester container. int jdiscMemoryPercentage = (int) Math.ceil(200.0 / memoryGb); // 2Gb memory for tester application (excessive?). - int testMemoryMb = 768 * (memoryGb - 2); // Memory allocated to Surefire running tests. ≥25% left for other stuff. + int testMemoryMb = 512 * (memoryGb - 2); // Memory allocated to Surefire running tests. ≥25% left for other stuff. String servicesXml = "<?xml version='1.0' encoding='UTF-8'?>\n" + "<services xmlns:deploy='vespa' version='1.0'>\n" + - " <container version='1.0' id='default'>\n" + + " <container version='1.0' id='tester'>\n" + "\n" + " <component id=\"com.yahoo.vespa.hosted.testrunner.TestRunner\" bundle=\"vespa-testrunner-components\">\n" + " <config name=\"com.yahoo.vespa.hosted.testrunner.test-runner\">\n" + @@ -646,7 +668,9 @@ public class InternalStepRunner implements StepRunner { " </filtering>\n" + " </http>\n" + "\n" + - " <nodes count=\"1\" flavor=\"" + flavor + "\" allocated-memory=\"" + jdiscMemoryPercentage + "%\" />\n" + + " <nodes count=\"1\" flavor=\"" + flavor + "\">\n" + + " <jvm allocated-memory=\"" + jdiscMemoryPercentage + "%\" />\n" + + " </nodes>\n" + " </container>\n" + "</services>\n"; @@ -664,38 +688,6 @@ public class InternalStepRunner implements StepRunner { return deploymentSpec.getBytes(StandardCharsets.UTF_8); } - /** Returns the config for the tests to run for the given job. */ - private static byte[] testConfig(ApplicationId id, ZoneId testerZone, SystemName system, - Map<ZoneId, List<URI>> deployments, Map<ZoneId, List<String>> clusters) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - root.setString("application", id.serializedForm()); - root.setString("zone", testerZone.value()); - root.setString("system", system.name()); - - Cursor endpointsObject = root.setObject("endpoints"); - deployments.forEach((zone, endpoints) -> { - Cursor endpointArray = endpointsObject.setArray(zone.value()); - for (URI endpoint : endpoints) - endpointArray.addString(endpoint.toString()); - }); - - Cursor clustersObject = root.setObject("clusters"); - clusters.forEach((zone, clusterList) -> { - Cursor clusterArray = clustersObject.setArray(zone.value()); - for (String cluster : clusterList) - clusterArray.addString(cluster); - }); - - try { - return SlimeUtils.toJsonBytes(slime); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - /** Logger which logs to a {@link JobController}, as well as to the parent class' {@link Logger}. */ private class DualLogger { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 245af60d0ad..c644af2e554 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -41,6 +42,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -72,6 +74,8 @@ public class JobController { private final TesterCloud cloud; private final Badges badges; + private AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { }); + public JobController(Controller controller, RunDataStore runDataStore, TesterCloud testerCloud) { this.controller = controller; this.curator = controller.curator(); @@ -82,6 +86,7 @@ public class JobController { public TesterCloud cloud() { return cloud; } public int historyLength() { return historyLength; } + public void setRunner(Consumer<Run> runner) { this.runner.set(runner); } /** Rewrite all job data with the newest format. */ public void updateStorage() { @@ -317,12 +322,16 @@ public class JobController { ApplicationVersion.unknown, Optional.empty(), Optional.empty())); + + runner.get().accept(last(id, type).get()); }); } /** Aborts a run and waits for it complete. */ private void abortAndWait(RunId id) { abort(id); + runner.get().accept(last(id.application(), id.type()).get()); + while ( ! last(id.application(), id.type()).get().hasEnded()) { try { Thread.sleep(100); @@ -399,9 +408,19 @@ public class JobController { /** Returns a URI of the tester endpoint retrieved from the routing generator, provided it matches an expected form. */ Optional<URI> testerEndpoint(RunId id) { - ApplicationId tester = id.tester().id(); - return controller.applications().getDeploymentEndpoints(new DeploymentId(tester, id.type().zone(controller.system()))) - .flatMap(uris -> uris.stream().findAny()); + DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); + return controller.applications().getDeploymentEndpoints(testerId) + .stream().findAny() + .or(() -> controller.applications().routingPolicies().get(testerId).stream() + .findAny() + .map(policy -> policy.endpointIn(controller.system()).url())); + } + + /** Returns a set containing the zone of the deployment tested in the given run, and all production zones for the application. */ + public Set<ZoneId> testedZoneAndProductionZones(ApplicationId id, JobType type) { + return Stream.concat(Stream.of(type.zone(controller.system())), + controller.applications().require(id).productionDeployments().keySet().stream()) + .collect(Collectors.toSet()); } // TODO jvenstad: Find a more appropriate way of doing this, at least when this is the only build service. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 0cb400004aa..f052c7d91ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -14,6 +14,7 @@ import java.util.Optional; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static java.util.Objects.requireNonNull; @@ -173,15 +174,14 @@ public class Run { .iterator()); } - /** Returns the list of not-yet-succeeded run-always steps whose run-always prerequisites have all succeeded. */ + /** Returns the list of not-yet-run run-always steps whose run-always prerequisites have all run. */ private List<Step> forcedSteps() { return ImmutableList.copyOf(steps.entrySet().stream() - .filter(entry -> entry.getValue() != succeeded + .filter(entry -> entry.getValue() == unfinished && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) && entry.getKey().prerequisites().stream() .filter(JobProfile.of(id.type()).alwaysRun()::contains) - .allMatch(step -> steps.get(step) == null - || steps.get(step) == succeeded)) + .allMatch(step -> steps.get(step) != unfinished)) .map(Map.Entry::getKey) .iterator()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 051a99074d1..a5f9ef86da4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -15,7 +15,7 @@ import java.util.List; * only the prerequisites of a step which are included in a run's profile will be considered. * Under normal circumstances, a step will run only after each of its prerequisites have succeeded. * When a run has failed, however, each of the always-run steps of the run's profile will be run, - * again in a topological order, and again requiring success of all their always-run prerequisites. + * again in a topological order, and requiring all their always-run prerequisites to have run. * * 2. A step will never run concurrently with its prerequisites. This is to ensure, e.g., that relevant * information from a failed run is stored, and that deployment does not occur after deactivation. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java new file mode 100644 index 00000000000..e79692d34ed --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -0,0 +1,82 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.util.List; +import java.util.Map; + +/** + * Serializes config for integration tests against Vespa deployments. + * + * @author jonmv + */ +public class TestConfigSerializer { + + private final SystemName system; + + public TestConfigSerializer(SystemName system) { + this.system = system; + } + + public Slime configSlime(ApplicationId id, + JobType type, + Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments, + Map<ZoneId, List<String>> clusters) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + root.setString("application", id.serializedForm()); + root.setString("zone", type.zone(system).value()); + root.setString("system", system.value()); + + Cursor endpointsObject = root.setObject("endpoints"); // TODO jvenstad: remove. + deployments.forEach((zone, endpoints) -> { + Cursor endpointArray = endpointsObject.setArray(zone.value()); + for (URI endpoint : endpoints.values()) + endpointArray.addString(endpoint.toString()); + }); + + Cursor zoneEndpointsObject = root.setObject("zoneEndpoints"); + deployments.forEach((zone, endpoints) -> { + Cursor clusterEndpointsObject = zoneEndpointsObject.setObject(zone.value()); + endpoints.forEach((cluster, endpoint) -> { + clusterEndpointsObject.setString(cluster.value(), endpoint.toString()); + }); + }); + + if ( ! clusters.isEmpty()) { + Cursor clustersObject = root.setObject("clusters"); + clusters.forEach((zone, clusterList) -> { + Cursor clusterArray = clustersObject.setArray(zone.value()); + for (String cluster : clusterList) + clusterArray.addString(cluster); + }); + } + + return slime; + } + + /** Returns the config for the tests to run for the given job. */ + public byte[] configJson(ApplicationId id, + JobType type, + Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments, + Map<ZoneId, List<String>> clusters) { + try { + return SlimeUtils.toJsonBytes(configSlime(id, type, deployments, clusters)); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java index b17ca52d835..3cd3d969731 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java @@ -43,11 +43,6 @@ public class CreateRecord implements NameServiceRequest { } @Override - public List<Record> affectedRecords() { - return List.of(record); - } - - @Override public String toString() { return "create record " + record; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java index ec943676962..9dd02735638 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java @@ -52,11 +52,6 @@ public class CreateRecords implements NameServiceRequest { } @Override - public List<Record> affectedRecords() { - return records(); - } - - @Override public String toString() { return "create records " + records(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index f4bea8b1083..4e461534cc0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -39,24 +39,24 @@ public class NameServiceForwarder { } /** Create or update a CNAME record with given name and data */ - public Record createCname(RecordName name, RecordData canonicalName, NameServiceQueue.Priority priority) { - return forward(new CreateRecord(new Record(Record.Type.CNAME, name, canonicalName)), priority).affectedRecords().get(0); + public void createCname(RecordName name, RecordData canonicalName, NameServiceQueue.Priority priority) { + forward(new CreateRecord(new Record(Record.Type.CNAME, name, canonicalName)), priority); } /** Create or update an ALIAS record with given name and targets */ - public List<Record> createAlias(RecordName name, Set<AliasTarget> targets, NameServiceQueue.Priority priority) { + public void createAlias(RecordName name, Set<AliasTarget> targets, NameServiceQueue.Priority priority) { var records = targets.stream().map(AliasTarget::asData) .map(data -> new Record(Record.Type.ALIAS, name, data)) .collect(Collectors.toList()); - return forward(new CreateRecords(records), priority).affectedRecords(); + forward(new CreateRecords(records), priority); } /** Create or update a TXT record with given name and data */ - public List<Record> createTxt(RecordName name, List<RecordData> txtData, NameServiceQueue.Priority priority) { + public void createTxt(RecordName name, List<RecordData> txtData, NameServiceQueue.Priority priority) { var records = txtData.stream() .map(data -> new Record(Record.Type.TXT, name, data)) .collect(Collectors.toList()); - return forward(new CreateRecords(records), priority).affectedRecords(); + forward(new CreateRecords(records), priority); } /** Remove all records of given type and name */ @@ -69,7 +69,7 @@ public class NameServiceForwarder { forward(new RemoveRecords(type, data), priority); } - private NameServiceRequest forward(NameServiceRequest request, NameServiceQueue.Priority priority) { + private void forward(NameServiceRequest request, NameServiceQueue.Priority priority) { try (Lock lock = db.lockNameServiceQueue()) { NameServiceQueue queue = db.readNameServiceQueue(); var queued = queue.requests().size(); @@ -78,9 +78,9 @@ public class NameServiceForwarder { "requests. This likely means that the name service is not successfully " + "executing requests"); } + log.log(LogLevel.INFO, "Queueing name service request: " + request); db.writeNameServiceQueue(queue.with(request, priority).last(maxQueuedRequests)); } - return request; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java index cc49e589cbb..4768577aa7b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java @@ -72,8 +72,9 @@ public class NameServiceQueue { if (requests.isEmpty()) return this; var queue = new NameServiceQueue(requests); - for (var i = 0; i < n && !queue.requests.isEmpty(); i++) { + for (int i = 0; i < n && !queue.requests.isEmpty(); i++) { var request = queue.requests.peek(); + log.log(LogLevel.INFO, "Dispatching name service request: " + request); try { request.dispatchTo(nameService); queue.requests.poll(); @@ -97,11 +98,13 @@ public class NameServiceQueue { /** Priority of a request added to this */ public enum Priority { + /** Default priority. Request will be delivered in FIFO order */ normal, - /**Request is queued before others. Useful for code that needs to act on effects of a request */ + /** Request is queued first. Useful for code that needs to act on effects of a request */ high + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java index a01719ccc88..65076694160 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java @@ -2,9 +2,6 @@ package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; - -import java.util.List; /** * Interface for requests to a {@link NameService}. @@ -16,7 +13,4 @@ public interface NameServiceRequest { /** Send this to given name service */ void dispatchTo(NameService nameService); - /** Returns the records affected by executing this */ - List<Record> affectedRecords(); - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java index b721f66e452..ddc4d157afd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java @@ -61,11 +61,6 @@ public class RemoveRecords implements NameServiceRequest { } @Override - public List<Record> affectedRecords() { - return List.of(); - } - - @Override public String toString() { return "remove records of type " + type + ", by " + name.map(n -> "name " + n).orElse("") + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java new file mode 100644 index 00000000000..c6956293adf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java @@ -0,0 +1,39 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; + +import java.time.Duration; +import java.util.EnumSet; + +/** + * @author olaa + */ +public class BillingMaintainer extends Maintainer { + + private final Billing billing; + + public BillingMaintainer(Controller controller, Duration interval, JobControl jobControl, Billing billing) { + super(controller, interval, jobControl, BillingMaintainer.class.getSimpleName(), EnumSet.of(SystemName.cd)); + this.billing = billing; + } + + @Override + public void maintain() { + controller().tenants().asList() + .stream() + .filter(tenant -> tenant instanceof CloudTenant) + .map(tenant -> (CloudTenant) tenant) + .forEach(cloudTenant -> controller().applications().asList(cloudTenant.name()) + .stream() + .forEach( application -> { + billing.handleBilling(application.id(), cloudTenant.billingInfo().customerId()); + }) + ); + } +} + + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index c9b07ada854..d9aaef6ef3b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -5,14 +5,12 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; -import java.io.IOException; import java.time.Duration; import java.util.HashMap; import java.util.List; @@ -22,8 +20,8 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Maintain info about hardware, hostnames and cluster specifications. - * <p> + * Maintains information about hardware, hostnames and cluster specifications. + * * This is used to calculate cost metrics for the application api. * * @author smorgrav @@ -33,26 +31,26 @@ public class ClusterInfoMaintainer extends Maintainer { private static final Logger log = Logger.getLogger(ClusterInfoMaintainer.class.getName()); private final Controller controller; - private final NodeRepositoryClientInterface nodeRepositoryClient; + private final NodeRepository nodeRepository; ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl, - NodeRepositoryClientInterface nodeRepositoryClient) { + NodeRepository nodeRepository) { super(controller, duration, jobControl); this.controller = controller; - this.nodeRepositoryClient = nodeRepositoryClient; + this.nodeRepository = nodeRepository; } - private static String clusterid(NodeRepositoryNode node) { + private static String clusterId(NodeRepositoryNode node) { return node.getMembership().clusterid; } - private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeList nodes, ZoneId zone) { + private Map<ClusterSpec.Id, ClusterInfo> getClusterInfo(NodeList nodes) { Map<ClusterSpec.Id, ClusterInfo> infoMap = new HashMap<>(); // Group nodes by clusterid Map<String, List<NodeRepositoryNode>> clusters = nodes.nodes().stream() .filter(node -> node.getMembership() != null) - .collect(Collectors.groupingBy(ClusterInfoMaintainer::clusterid)); + .collect(Collectors.groupingBy(ClusterInfoMaintainer::clusterId)); // For each cluster - get info for (String id : clusters.keySet()) { @@ -65,20 +63,11 @@ public class ClusterInfoMaintainer extends Maintainer { double cpu = 0; double mem = 0; double disk = 0; - // TODO: This code was never run. Reenable when flavours are available from a FlavorRegistry or something, or remove. - /*if (zone.nodeFlavors().isPresent()) { - Optional<Flavor> flavorOptional = zone.nodeFlavors().get().getFlavor(node.flavor); - if ((flavorOptional.isPresent())) { - Flavor flavor = flavorOptional.get(); - cpu = flavor.getMinCpuCores(); - mem = flavor.getMinMainMemoryAvailableGb(); - disk = flavor.getMinMainMemoryAvailableGb(); - } - }*/ // Add to map List<String> hostnames = clusterNodes.stream().map(NodeRepositoryNode::getHostname).collect(Collectors.toList()); - ClusterInfo inf = new ClusterInfo(node.getFlavor(), node.getCost(), cpu, mem, disk, + int cost = node.getCost() == null ? 0 : node.getCost(); // Cost is not guaranteed to be defined for all flavors + ClusterInfo inf = new ClusterInfo(node.getFlavor(), cost, cpu, mem, disk, ClusterSpec.Type.from(node.getMembership().clustertype), hostnames); infoMap.put(new ClusterSpec.Id(id), inf); } @@ -92,16 +81,12 @@ public class ClusterInfoMaintainer extends Maintainer { for (Deployment deployment : application.deployments().values()) { DeploymentId deploymentId = new DeploymentId(application.id(), deployment.zone()); try { - NodeList nodes = nodeRepositoryClient.listNodes(deploymentId.zoneId(), - deploymentId.applicationId().tenant().value(), - deploymentId.applicationId().application().value(), - deploymentId.applicationId().instance().value()); - Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes, deployment.zone()); + NodeList nodes = nodeRepository.listNodes(deploymentId.zoneId(), deploymentId.applicationId()); + Map<ClusterSpec.Id, ClusterInfo> clusterInfo = getClusterInfo(nodes); controller().applications().lockIfPresent(application.id(), lockedApplication -> controller.applications().store(lockedApplication.withClusterInfo(deployment.zone(), clusterInfo))); - } - catch (IOException | IllegalArgumentException e) { - log.log(Level.WARNING, "Failing getting cluster info of for " + deploymentId, e); + } catch (Exception e) { + log.log(Level.WARNING, "Failing getting cluster information for " + deploymentId, e); } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index 7955505a2b0..7cf710623ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import java.time.Duration; import java.util.HashMap; import java.util.Map; +import java.util.function.Predicate; /** * Fetch utilization metrics and update applications with this data. @@ -24,7 +26,7 @@ public class ClusterUtilizationMaintainer extends Maintainer { private final Controller controller; public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) { - super(controller, duration, jobControl); + super(controller, duration, jobControl, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.controller = controller; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 7dfbc135df9..0080d7c23c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -11,9 +11,9 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.EnumSet; import java.util.Objects; import java.util.Optional; +import java.util.function.Predicate; import java.util.logging.Logger; /** @@ -28,7 +28,7 @@ public class ContactInformationMaintainer extends Maintainer { private final ContactRetriever contactRetriever; public ContactInformationMaintainer(Controller controller, Duration interval, JobControl jobControl, ContactRetriever contactRetriever) { - super(controller, interval, jobControl, null, EnumSet.of(SystemName.cd, SystemName.main)); + super(controller, interval, jobControl, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.contactRetriever = Objects.requireNonNull(contactRetriever, "organization must be non-null"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index aec4c7a915c..116ea532a11 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -2,17 +2,17 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; -import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; +import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; -import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; +import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; @@ -46,49 +46,48 @@ public class ControllerMaintenance extends AbstractComponent { private final ClusterUtilizationMaintainer clusterUtilizationMaintainer; private final DeploymentMetricsMaintainer deploymentMetricsMaintainer; private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer; - private final DnsMaintainer dnsMaintainer; private final SystemUpgrader systemUpgrader; private final List<OsUpgrader> osUpgraders; private final OsVersionStatusUpdater osVersionStatusUpdater; private final JobRunner jobRunner; private final ContactInformationMaintainer contactInformationMaintainer; private final CostReportMaintainer costReportMaintainer; - private final RoutingPolicyMaintainer routingPolicyMaintainer; private final ResourceMeterMaintainer resourceMeterMaintainer; private final NameServiceDispatcher nameServiceDispatcher; + private final BillingMaintainer billingMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, - JobControl jobControl, Metric metric, Chef chefClient, + JobControl jobControl, Metric metric, DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, - NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient, + NameService nameService, NodeRepository nodeRepository, ContactRetriever contactRetriever, CostReportConsumer reportConsumer, ResourceSnapshotConsumer resourceSnapshotConsumer, + Billing billing, SelfHostedCostConfig selfHostedCostConfig) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); this.jobControl = jobControl; deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl); deploymentIssueReporter = new DeploymentIssueReporter(controller, deploymentIssues, maintenanceInterval, jobControl); - metricsReporter = new MetricsReporter(controller, metric, chefClient, jobControl, controller.system()); + metricsReporter = new MetricsReporter(controller, metric, jobControl); outstandingChangeDeployer = new OutstandingChangeDeployer(controller, Duration.ofMinutes(1), jobControl); versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(1), jobControl); upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofMinutes(1), jobControl); - clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient); + clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepository); clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); - dnsMaintainer = new DnsMaintainer(controller, Duration.ofMinutes(5), jobControl); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); jobRunner = new JobRunner(controller, Duration.ofMinutes(2), jobControl); osUpgraders = osUpgraders(controller, jobControl); osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever); - costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); - routingPolicyMaintainer = new RoutingPolicyMaintainer(controller, Duration.ofMinutes(5), jobControl, curator); - resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(60), jobControl, nodeRepositoryClient, Clock.systemUTC(), metric, resourceSnapshotConsumer); + costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepository, Clock.systemUTC(), selfHostedCostConfig); + resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(60), jobControl, nodeRepository, Clock.systemUTC(), metric, resourceSnapshotConsumer); nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl, nameService); + billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billing); } public Upgrader upgrader() { return upgrader; } @@ -109,22 +108,21 @@ public class ControllerMaintenance extends AbstractComponent { clusterInfoMaintainer.deconstruct(); deploymentMetricsMaintainer.deconstruct(); applicationOwnershipConfirmer.deconstruct(); - dnsMaintainer.deconstruct(); systemUpgrader.deconstruct(); osUpgraders.forEach(Maintainer::deconstruct); osVersionStatusUpdater.deconstruct(); jobRunner.deconstruct(); contactInformationMaintainer.deconstruct(); costReportMaintainer.deconstruct(); - routingPolicyMaintainer.deconstruct(); resourceMeterMaintainer.deconstruct(); nameServiceDispatcher.deconstruct(); + billingMaintainer.deconstruct(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ private static List<OsUpgrader> osUpgraders(Controller controller, JobControl jobControl) { - return controller.zoneRegistry().zones().controllerUpgraded().ids().stream() - .map(ZoneId::cloud) + return controller.zoneRegistry().zones().controllerUpgraded().zones().stream() + .map(ZoneApi::getCloudName) .distinct() .sorted() .map(cloud -> new OsUpgrader(controller, Duration.ofMinutes(1), jobControl, cloud)) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 2b26e93aeb8..f3dac2be22e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -5,14 +5,15 @@ import com.google.inject.Inject; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.restapi.cost.CostCalculator; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import java.time.Clock; import java.time.Duration; -import java.util.*; +import java.util.EnumSet; +import java.util.Objects; import java.util.logging.Logger; /** @@ -26,7 +27,7 @@ public class CostReportMaintainer extends Maintainer { private static final Logger log = Logger.getLogger(CostReportMaintainer.class.getName()); private final CostReportConsumer consumer; - private final NodeRepositoryClientInterface nodeRepository; + private final NodeRepository nodeRepository; private final Clock clock; private final SelfHostedCostConfig selfHostedCostConfig; @@ -35,7 +36,7 @@ public class CostReportMaintainer extends Maintainer { public CostReportMaintainer(Controller controller, Duration interval, CostReportConsumer consumer, JobControl jobControl, - NodeRepositoryClientInterface nodeRepository, + NodeRepository nodeRepository, Clock clock, SelfHostedCostConfig selfHostedCostConfig) { super(controller, interval, jobControl, "CostReportMaintainer", EnumSet.of(SystemName.main)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 3e07d0879ea..1d9f76cddcd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -22,6 +23,7 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; @@ -41,7 +43,8 @@ public class DeploymentMetricsMaintainer extends Maintainer { private final ApplicationController applications; public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) { - super(controller, duration, jobControl); + super(controller, duration, jobControl, DeploymentMetricsMaintainer.class.getSimpleName(), + SystemName.allOf(Predicate.not(SystemName::isPublic))); this.applications = controller.applications(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java deleted file mode 100644 index 7e0032f03c5..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.rotation.Rotation; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Performs DNS maintenance tasks such as removing DNS aliases for unassigned rotations. - * - * @author mpolden - */ -public class DnsMaintainer extends Maintainer { - - private final AtomicInteger rotationIndex = new AtomicInteger(0); - - public DnsMaintainer(Controller controller, Duration interval, JobControl jobControl) { - super(controller, interval, jobControl); - } - - private RotationRepository rotationRepository() { - return controller().applications().rotationRepository(); - } - - @Override - protected void maintain() { - try (RotationLock lock = rotationRepository().lock()) { - Map<RotationId, Rotation> unassignedRotations = rotationRepository().availableRotations(lock); - rotationToCheckOf(unassignedRotations.values()).ifPresent(this::removeCname); - } - } - - /** Remove CNAME(s) for unassigned rotation */ - private void removeCname(Rotation rotation) { - // When looking up CNAME by data, the data must be a FQDN - controller().nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordData.fqdn(rotation.name()), Priority.normal); - } - - /** - * Returns the rotation that should be checked in this run. We check only one rotation per run to avoid running into - * rate limits that may be imposed by the {@link NameService} implementation. - */ - private Optional<Rotation> rotationToCheckOf(Collection<Rotation> rotations) { - if (rotations.isEmpty()) return Optional.empty(); - List<Rotation> rotationList = new ArrayList<>(rotations); - int index = rotationIndex.getAndUpdate((i) -> { - if (i < rotationList.size() - 1) { - return ++i; - } - return 0; - }); - return Optional.of(rotationList.get(index)); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 0333711ae39..b8bb9a7ef79 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.config.provision.zone.UpgradePolicy; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.yolean.Exceptions; @@ -42,9 +42,9 @@ public abstract class InfrastructureUpgrader extends Maintainer { /** Deploy a list of system applications until they converge on the given version */ private void upgradeAll(Version target, List<SystemApplication> applications) { - for (List<ZoneId> zones : upgradePolicy.asList()) { + for (List<ZoneApi> zones : upgradePolicy.asList()) { boolean converged = true; - for (ZoneId zone : zones) { + for (ZoneApi zone : zones) { try { converged &= upgradeAll(target, applications, zone); } catch (UnreachableNodeRepositoryException e) { @@ -62,39 +62,44 @@ public abstract class InfrastructureUpgrader extends Maintainer { } /** Returns whether all applications have converged to the target version in zone */ - private boolean upgradeAll(Version target, List<SystemApplication> applications, ZoneId zone) { + private boolean upgradeAll(Version target, List<SystemApplication> applications, ZoneApi zone) { boolean converged = true; for (SystemApplication application : applications) { if (convergedOn(target, application.dependencies(), zone)) { - upgrade(target, application, zone); + boolean currentAppConverged = convergedOn(target, application, zone); + // In dynamically provisioned zones there may be no tenant hosts at the time of upgrade, so we + // should always set the target version. + if (application == SystemApplication.tenantHost || !currentAppConverged) { + upgrade(target, application, zone); + } + converged &= currentAppConverged; } - converged &= convergedOn(target, application, zone); } return converged; } - private boolean convergedOn(Version target, List<SystemApplication> applications, ZoneId zone) { + private boolean convergedOn(Version target, List<SystemApplication> applications, ZoneApi zone) { return applications.stream().allMatch(application -> convergedOn(target, application, zone)); } /** Upgrade component to target version. Implementation should be idempotent */ - protected abstract void upgrade(Version target, SystemApplication application, ZoneId zone); + protected abstract void upgrade(Version target, SystemApplication application, ZoneApi zone); /** Returns whether application has converged to target version in zone */ - protected abstract boolean convergedOn(Version target, SystemApplication application, ZoneId zone); + protected abstract boolean convergedOn(Version target, SystemApplication application, ZoneApi zone); /** Returns the target version for the component upgraded by this, if any */ protected abstract Optional<Version> targetVersion(); /** Returns whether the upgrader should require given node to upgrade */ - protected abstract boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone); + protected abstract boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone); /** Find the minimum value of a version field in a zone */ - protected final Optional<Version> minVersion(ZoneId zone, SystemApplication application, Function<Node, Version> versionField) { + protected final Optional<Version> minVersion(ZoneApi zone, SystemApplication application, Function<Node, Version> versionField) { try { return controller().configServer() .nodeRepository() - .list(zone, application.id()) + .list(zone.getId(), application.id()) .stream() .filter(node -> requireUpgradeOf(node, application, zone)) .map(versionField) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 47e17b34d2a..f13c31de5d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -34,7 +34,6 @@ public class JobRunner extends Maintainer { private final ExecutorService executors; private final StepRunner runner; - @Inject public JobRunner(Controller controller, Duration duration, JobControl jobControl) { this(controller, duration, jobControl, Executors.newFixedThreadPool(32), new InternalStepRunner(controller)); } @@ -43,6 +42,7 @@ public class JobRunner extends Maintainer { public JobRunner(Controller controller, Duration duration, JobControl jobControl, ExecutorService executors, StepRunner runner) { super(controller, duration, jobControl); this.jobs = controller.jobController(); + this.jobs.setRunner(this::advance); this.executors = executors; this.runner = runner; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 449b8c51acd..c7b76696d84 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -3,14 +3,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.SystemName; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -24,10 +19,8 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; /** @@ -36,7 +29,6 @@ import java.util.stream.Collectors; */ public class MetricsReporter extends Maintainer { - public static final String CONVERGENCE_METRIC = "seconds.since.last.chef.convergence"; public static final String DEPLOYMENT_FAIL_METRIC = "deployment.failurePercentage"; public static final String DEPLOYMENT_AVERAGE_DURATION = "deployment.averageDuration"; public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades"; @@ -46,27 +38,16 @@ public class MetricsReporter extends Maintainer { public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; private final Metric metric; - private final Chef chefClient; private final Clock clock; - private final SystemName system; - public MetricsReporter(Controller controller, Metric metric, Chef chefClient, JobControl jobControl, - SystemName system) { - this(controller, metric, chefClient, Clock.systemUTC(), jobControl, system); - } - - public MetricsReporter(Controller controller, Metric metric, Chef chefClient, Clock clock, - JobControl jobControl, SystemName system) { + public MetricsReporter(Controller controller, Metric metric, JobControl jobControl) { super(controller, Duration.ofMinutes(1), jobControl); // use fixed rate for metrics this.metric = metric; - this.chefClient = chefClient; - this.clock = clock; - this.system = system; + this.clock = controller.clock(); } @Override public void maintain() { - reportChefMetrics(); reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); @@ -79,49 +60,6 @@ public class MetricsReporter extends Maintainer { } } - private void reportChefMetrics() { - String query = "chef_environment:hosted*"; - if (system == SystemName.cd) { - query += " AND hosted_system:" + system; - } - PartialNodeResult nodeResult = chefClient.partialSearchNodes(query, - List.of( - AttributeMapping.simpleMapping("fqdn"), - AttributeMapping.simpleMapping("ohai_time"), - AttributeMapping.deepMapping("tenant", List.of("hosted", "owner", "tenant")), - AttributeMapping.deepMapping("application", List.of("hosted", "owner", "application")), - AttributeMapping.deepMapping("instance", List.of("hosted", "owner", "instance")), - AttributeMapping.deepMapping("environment", List.of("hosted", "environment")), - AttributeMapping.deepMapping("region", List.of("hosted", "region")), - AttributeMapping.deepMapping("system", List.of("hosted", "system")) - )); - - // The above search will return a correct list if the system is CD. However for main, it will - // return all nodes, since system==nil for main - keepNodesWithSystem(nodeResult, system); - - Instant instant = clock.instant(); - for (PartialNode node : nodeResult.rows) { - String hostname = node.getFqdn(); - long secondsSinceConverge = Duration.between(Instant.ofEpochSecond(node.getOhaiTime().longValue()), instant).getSeconds(); - Map<String, String> dimensions = new HashMap<>(); - dimensions.put("host", hostname); - dimensions.put("system", node.getValue("system").orElse("main")); - Optional<String> environment = node.getValue("environment"); - Optional<String> region = node.getValue("region"); - - if (environment.isPresent() && region.isPresent()) { - dimensions.put("zone", String.format("%s.%s", environment.get(), region.get())); - } - - node.getValue("tenant").ifPresent(tenant -> dimensions.put("tenantName", tenant)); - Optional<String> application = node.getValue("application"); - application.ifPresent(app -> dimensions.put("app", String.format("%s.%s", app, node.getValue("instance").orElse("default")))); - Metric.Context context = metric.createContext(dimensions); - metric.set(CONVERGENCE_METRIC, secondsSinceConverge, context); - } - } - private void reportDeploymentMetrics() { ApplicationList applications = ApplicationList.from(controller().applications().asList()) .hasProductionDeployment(); @@ -210,10 +148,6 @@ public class MetricsReporter extends Maintainer { .max(Integer::compareTo) .orElse(0); } - - private static void keepNodesWithSystem(PartialNodeResult nodeResult, SystemName system) { - nodeResult.rows.removeIf(node -> !system.name().equals(node.getValue("system").orElse("main"))); - } private static Map<String, String> dimensions(ApplicationId application) { return ImmutableMap.of( diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java index 8878ac9bd5b..d4dc068c71f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java @@ -41,6 +41,7 @@ public class NameServiceDispatcher extends Maintainer { try (Lock lock = db.lockNameServiceQueue()) { NameServiceQueue queue = db.readNameServiceQueue(); NameServiceQueue remaining = queue.dispatchTo(nameService, requestCount); + if (queue == remaining) return; // Queue unchanged db.writeNameServiceQueue(remaining); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 3b521657f15..8845f4c652f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.versions.OsVersion; @@ -38,23 +38,22 @@ public class OsUpgrader extends InfrastructureUpgrader { } @Override - protected void upgrade(Version target, SystemApplication application, ZoneId zone) { - if (wantedVersion(zone, application, target).equals(target)) { + protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { + if (!application.isEligibleForOsUpgrades() || wantedVersion(zone, application, target).equals(target)) { return; } - log.info(String.format("Upgrading OS of %s to version %s in %s", application.id(), target, zone)); - application.nodeTypesWithUpgradableOs().forEach(nodeType -> controller().configServer().nodeRepository() - .upgradeOs(zone, nodeType, target)); + log.info(String.format("Upgrading OS of %s to version %s in %s in cloud %s", application.id(), target, zone.getId(), zone.getCloudName())); + controller().configServer().nodeRepository().upgradeOs(zone.getId(), application.nodeType(), target); } @Override - protected boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { + protected boolean convergedOn(Version target, SystemApplication application, ZoneApi zone) { return currentVersion(zone, application, target).equals(target); } @Override - protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone) { - return cloud.equals(zone.cloud()) && eligibleForUpgrade(node, application); + protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { + return cloud.equals(zone.getCloudName()) && eligibleForUpgrade(node, application); } @Override @@ -66,18 +65,18 @@ public class OsUpgrader extends InfrastructureUpgrader { .map(OsVersion::version); } - private Version currentVersion(ZoneId zone, SystemApplication application, Version defaultVersion) { + private Version currentVersion(ZoneApi zone, SystemApplication application, Version defaultVersion) { return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion); } - private Version wantedVersion(ZoneId zone, SystemApplication application, Version defaultVersion) { + private Version wantedVersion(ZoneApi zone, SystemApplication application, Version defaultVersion) { return minVersion(zone, application, Node::wantedOsVersion).orElse(defaultVersion); } /** Returns whether node in application should be upgraded by this */ public static boolean eligibleForUpgrade(Node node, SystemApplication application) { return upgradableNodeStates.contains(node.state()) && - application.nodeTypesWithUpgradableOs().contains(node.type()); + application.isEligibleForOsUpgrades(); } private static String name(CloudName cloud) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 5ed7a14836e..5e2ba48da3f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,25 +1,28 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; import java.util.stream.Collectors; -import static com.yahoo.yolean.Exceptions.uncheck; - /** * Creates a ResourceSnapshot per application, which is then passed on to a ResourceSnapshotConsumer * TODO: Write JSON blob of node repo somewhere @@ -30,7 +33,7 @@ public class ResourceMeterMaintainer extends Maintainer { private final Clock clock; private final Metric metric; - private final NodeRepositoryClientInterface nodeRepository; + private final NodeRepository nodeRepository; private final ResourceSnapshotConsumer resourceSnapshotConsumer; private static final String metering_last_reported = "metering_last_reported"; @@ -40,11 +43,11 @@ public class ResourceMeterMaintainer extends Maintainer { public ResourceMeterMaintainer(Controller controller, Duration interval, JobControl jobControl, - NodeRepositoryClientInterface nodeRepository, + NodeRepository nodeRepository, Clock clock, Metric metric, ResourceSnapshotConsumer resourceSnapshotConsumer) { - super(controller, interval, jobControl, ResourceMeterMaintainer.class.getSimpleName(), Set.of(SystemName.cd)); + super(controller, interval, jobControl, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.clock = clock; this.nodeRepository = nodeRepository; this.metric = metric; @@ -77,24 +80,19 @@ public class ResourceMeterMaintainer extends Maintainer { private List<NodeRepositoryNode> getNodes() { return controller().zoneRegistry().zones() - .reachable().ids().stream() - .flatMap(zoneId -> uncheck(() -> nodeRepository.listNodes(zoneId, true).nodes().stream())) + .ofCloud(CloudName.from("aws")) + .reachable().zones().stream() + .flatMap(zone -> nodeRepository.listNodes(zone.getId()).nodes().stream()) .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) + .filter(node -> node.getState() == NodeState.active) .collect(Collectors.toList()); } private Map<ApplicationId, ResourceAllocation> getResourceAllocationByApplication(List<NodeRepositoryNode> nodes) { - Map<ApplicationId, List<NodeRepositoryNode>> applicationNodes = new HashMap<>(); - - nodes.stream().forEach(node -> applicationNodes.computeIfAbsent(applicationIdFromNodeOwner(node.getOwner()), n -> new ArrayList<>()).add(node)); - - return applicationNodes.entrySet().stream() - .collect( - Collectors.toMap( - entry -> entry.getKey(), - entry -> ResourceAllocation.from(entry.getValue()) - ) - ); + return nodes.stream() + .collect(Collectors.groupingBy( + node -> applicationIdFromNodeOwner(node.getOwner()), + Collectors.collectingAndThen(Collectors.toList(), ResourceAllocation::from))); } private ApplicationId applicationIdFromNodeOwner(NodeOwner owner) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java new file mode 100644 index 00000000000..4a98cb49227 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java @@ -0,0 +1,198 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.RoutingId; +import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Updates routing policies and their associated DNS records based on an deployment's load balancers. + * + * @author mortent + * @author mpolden + */ +public class RoutingPolicies { + + private final Controller controller; + private final CuratorDb db; + + public RoutingPolicies(Controller controller) { + this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + this.db = controller.curator(); + try (var lock = db.lockRoutingPolicies()) { // Update serialized format + for (var policy : db.readRoutingPolicies().entrySet()) { + db.writeRoutingPolicies(policy.getKey(), policy.getValue()); + } + } + } + + /** Read all known routing policies for given application */ + public Set<RoutingPolicy> get(ApplicationId application) { + return db.readRoutingPolicies(application); + } + + /** Read all known routing policies for given deployment */ + public Set<RoutingPolicy> get(DeploymentId deployment) { + return get(deployment.applicationId(), deployment.zoneId()); + } + + /** Read all known routing policies for given deployment */ + public Set<RoutingPolicy> get(ApplicationId application, ZoneId zone) { + return db.readRoutingPolicies(application).stream() + .filter(policy -> policy.zone().equals(zone)) + .collect(Collectors.toUnmodifiableSet()); + } + + /** + * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if + * load balancers for given application have changed. + */ + public void refresh(ApplicationId application, ZoneId zone) { + // TODO: Use this to decide how apply routing policies for shared routing layer + if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; + var lbs = new LoadBalancers(application, zone, controller.applications().configServer() + .getLoadBalancers(application, zone)); + try (var lock = db.lockRoutingPolicies()) { + removeObsoleteEndpointsFromDns(lbs, lock); + storePoliciesOf(lbs, lock); + removeObsoletePolicies(lbs, lock); + registerEndpointsInDns(lbs, lock); + } + } + + /** Create global endpoints for given route, if any */ + private void registerEndpointsInDns(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(get(loadBalancers.application)); + + // Create DNS record for each routing ID + for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { + Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().rotation(), + controller.system()); + Set<AliasTarget> targets = routeEntry.getValue() + .stream() + .filter(policy -> policy.dnsZone().isPresent()) + .map(policy -> new AliasTarget(policy.canonicalName(), + policy.dnsZone().get(), + policy.zone())) + .collect(Collectors.toSet()); + controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); + } + } + + /** Store routing policies for given route */ + private void storePoliciesOf(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + Set<RoutingPolicy> policies = new LinkedHashSet<>(get(loadBalancers.application)); + for (LoadBalancer loadBalancer : loadBalancers.list) { + RoutingPolicy policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer); + if (!policies.add(policy)) { + policies.remove(policy); + policies.add(policy); + } + } + db.writeRoutingPolicies(loadBalancers.application, policies); + } + + /** Create a policy for given load balancer and register a CNAME for it */ + private RoutingPolicy createPolicy(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { + RoutingPolicy routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, + loadBalancer.hostname(), loadBalancer.dnsZone(), + loadBalancer.rotations()); + RecordName name = RecordName.from(routingPolicy.endpointIn(controller.system()).dnsName()); + RecordData data = RecordData.fqdn(loadBalancer.hostname().value()); + controller.nameServiceForwarder().createCname(name, data, Priority.normal); + return routingPolicy; + } + + /** Remove obsolete policies for given route and their CNAME records */ + private void removeObsoletePolicies(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var allPolicies = new LinkedHashSet<>(get(loadBalancers.application)); + var removalCandidates = new HashSet<>(allPolicies); + var activeLoadBalancers = loadBalancers.list.stream() + .map(LoadBalancer::hostname) + .collect(Collectors.toSet()); + // Remove active load balancers and irrelevant zones from candidates + removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) || + !policy.zone().equals(loadBalancers.zone)); + for (var policy : removalCandidates) { + var dnsName = policy.endpointIn(controller.system()).dnsName(); + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); + allPolicies.remove(policy); + } + db.writeRoutingPolicies(loadBalancers.application, allPolicies); + } + + /** Remove unreferenced global endpoints for given route from DNS */ + private void removeObsoleteEndpointsFromDns(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var zonePolicies = get(loadBalancers.application, loadBalancers.zone); + var removalCandidates = routingTableFrom(zonePolicies).keySet(); + var activeRoutingIds = routingIdsFrom(loadBalancers.list); + removalCandidates.removeAll(activeRoutingIds); + for (var id : removalCandidates) { + Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller.system()); + controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); + } + } + + /** Compute routing IDs from given load balancers */ + private static Set<RoutingId> routingIdsFrom(List<LoadBalancer> loadBalancers) { + Set<RoutingId> routingIds = new LinkedHashSet<>(); + for (var loadBalancer : loadBalancers) { + for (var rotation : loadBalancer.rotations()) { + routingIds.add(new RoutingId(loadBalancer.application(), rotation)); + } + } + return Collections.unmodifiableSet(routingIds); + } + + /** Compute a routing table from given policies */ + private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Set<RoutingPolicy> routingPolicies) { + var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); + for (var policy : routingPolicies) { + for (var rotation : policy.rotations()) { + var id = new RoutingId(policy.owner(), rotation); + routingTable.putIfAbsent(id, new ArrayList<>()); + routingTable.get(id).add(policy); + } + } + return routingTable; + } + + /** Load balancers for a particular deployment */ + private static class LoadBalancers { + + private final ApplicationId application; + private final ZoneId zone; + private final List<LoadBalancer> list; + + private LoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> list) { + this.application = application; + this.zone = zone; + this.list = list; + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java deleted file mode 100644 index 0ddc24147ee..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.RotationName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.RoutingId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Maintains routing policies and their DNS records for all exclusive load balancers in this system. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicyMaintainer extends Maintainer { - - private static final Logger log = Logger.getLogger(RoutingPolicyMaintainer.class.getName()); - - private final NameServiceForwarder nameServiceForwarder; - private final CuratorDb db; - - public RoutingPolicyMaintainer(Controller controller, - Duration interval, - JobControl jobControl, - CuratorDb db) { - super(controller, interval, jobControl); - this.nameServiceForwarder = controller.nameServiceForwarder(); - this.db = db; - // Update serialized format - try (Lock lock = db.lockRoutingPolicies()) { - for (var policy : db.readRoutingPolicies().entrySet()) { - db.writeRoutingPolicies(policy.getKey(), policy.getValue()); - } - } - } - - @Override - protected void maintain() { - Map<DeploymentId, List<LoadBalancer>> loadBalancers = findLoadBalancers(); - removeObsoleteEndpointsFromDns(loadBalancers); - storePolicies(loadBalancers); - removeObsoletePolicies(loadBalancers); - registerEndpointsInDns(); - } - - /** Find all exclusive load balancers in this system, grouped by deployment */ - private Map<DeploymentId, List<LoadBalancer>> findLoadBalancers() { - Map<DeploymentId, List<LoadBalancer>> result = new LinkedHashMap<>(); - for (ZoneId zone : controller().zoneRegistry().zones().controllerUpgraded().ids()) { - List<LoadBalancer> loadBalancers = controller().applications().configServer().getLoadBalancers(zone); - for (LoadBalancer loadBalancer : loadBalancers) { - DeploymentId deployment = new DeploymentId(loadBalancer.application(), zone); - result.compute(deployment, (k, existing) -> { - if (existing == null) { - existing = new ArrayList<>(); - } - existing.add(loadBalancer); - return existing; - }); - } - } - return Collections.unmodifiableMap(result); - } - - /** Create global endpoints for all current routing policies */ - private void registerEndpointsInDns() { - try (Lock lock = db.lockRoutingPolicies()) { - Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(db.readRoutingPolicies()); - - // Create DNS record for each routing ID - for (Map.Entry<RoutingId, List<RoutingPolicy>> route : routingTable.entrySet()) { - Endpoint endpoint = RoutingPolicy.endpointOf(route.getKey().application(), route.getKey().rotation(), - controller().system()); - Set<AliasTarget> targets = route.getValue() - .stream() - .filter(policy -> policy.dnsZone().isPresent()) - .map(policy -> new AliasTarget(policy.canonicalName(), - policy.dnsZone().get(), - policy.zone())) - .collect(Collectors.toSet()); - try { - nameServiceForwarder.createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); - } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to create or update DNS record for global rotation " + - endpoint.dnsName() + ". Retrying in " + maintenanceInterval(), e); - } - } - } - } - - /** Store routing policies for all load balancers */ - private void storePolicies(Map<DeploymentId, List<LoadBalancer>> loadBalancers) { - for (Map.Entry<DeploymentId, List<LoadBalancer>> entry : loadBalancers.entrySet()) { - ApplicationId application = entry.getKey().applicationId(); - ZoneId zone = entry.getKey().zoneId(); - try (Lock lock = db.lockRoutingPolicies()) { - Set<RoutingPolicy> policies = new LinkedHashSet<>(db.readRoutingPolicies(application)); - for (LoadBalancer loadBalancer : entry.getValue()) { - try { - RoutingPolicy policy = storePolicy(application, zone, loadBalancer); - if (!policies.add(policy)) { - policies.remove(policy); - policies.add(policy); - } - } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to create or update DNS record for load balancer " + - loadBalancer.hostname() + ". Retrying in " + maintenanceInterval(), - e); - } - } - db.writeRoutingPolicies(application, policies); - } - } - } - - /** Store policy for given load balancer and request a CNAME for it */ - private RoutingPolicy storePolicy(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { - RoutingPolicy routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, - loadBalancer.hostname(), loadBalancer.dnsZone(), - loadBalancer.rotations()); - RecordName name = RecordName.from(routingPolicy.endpointIn(controller().system()).dnsName()); - RecordData data = RecordData.fqdn(loadBalancer.hostname().value()); - nameServiceForwarder.createCname(name, data, Priority.normal); - return routingPolicy; - } - - /** Remove obsolete policies and their CNAME records */ - private void removeObsoletePolicies(Map<DeploymentId, List<LoadBalancer>> loadBalancers) { - try (Lock lock = db.lockRoutingPolicies()) { - var allPolicies = new HashMap<>(db.readRoutingPolicies()); - var removalCandidates = allPolicies.values().stream() - .flatMap(Collection::stream) - .collect(Collectors.toSet()); - var activeLoadBalancers = loadBalancers.values().stream() - .flatMap(Collection::stream) - .map(LoadBalancer::hostname) - .collect(Collectors.toSet()); - // Keep active load balancers by removing them from candidates - removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName())); - for (var policy : removalCandidates) { - var dnsName = policy.endpointIn(controller().system()).dnsName(); - nameServiceForwarder.removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); - // Remove stale policy from curator - var updatedPolicies = new LinkedHashSet<>(allPolicies.getOrDefault(policy.owner(), Set.of())); - updatedPolicies.remove(policy); - allPolicies.put(policy.owner(), updatedPolicies); - db.writeRoutingPolicies(policy.owner(), updatedPolicies); - } - } - } - - /** Remove DNS for global endpoints not referenced by given load balancers */ - private void removeObsoleteEndpointsFromDns(Map<DeploymentId, List<LoadBalancer>> loadBalancers) { - try (Lock lock = db.lockRoutingPolicies()) { - Set<RoutingId> removalCandidates = routingTableFrom(db.readRoutingPolicies()).keySet(); - Set<RoutingId> activeRoutingIds = routingIdsFrom(loadBalancers); - removalCandidates.removeAll(activeRoutingIds); - for (RoutingId id : removalCandidates) { - Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller().system()); - nameServiceForwarder.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); - } - } - } - - /** Compute routing IDs from given load balancers */ - private static Set<RoutingId> routingIdsFrom(Map<DeploymentId, List<LoadBalancer>> loadBalancers) { - Set<RoutingId> routingIds = new LinkedHashSet<>(); - for (List<LoadBalancer> values : loadBalancers.values()) { - for (LoadBalancer loadBalancer : values) { - for (RotationName rotation : loadBalancer.rotations()) { - routingIds.add(new RoutingId(loadBalancer.application(), rotation)); - } - } - } - return Collections.unmodifiableSet(routingIds); - } - - /** Compute a routing table from given policies */ - private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Map<ApplicationId, Set<RoutingPolicy>> routingPolicies) { - var flattenedPolicies = routingPolicies.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); - var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); - for (var policy : flattenedPolicies) { - for (var rotation : policy.rotations()) { - var id = new RoutingId(policy.owner(), rotation); - routingTable.compute(id, (k, policies) -> { - if (policies == null) { - policies = new ArrayList<>(); - } - policies.add(policy); - return policies; - }); - } - } - return routingTable; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 62d401cf478..156e8d0d242 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -30,23 +30,26 @@ public class SystemUpgrader extends InfrastructureUpgrader { } @Override - protected void upgrade(Version target, SystemApplication application, ZoneId zone) { + protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { if (minVersion(zone, application, Node::wantedVersion).map(target::isAfter) .orElse(true)) { - log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone)); - controller().applications().deploy(application, zone, target); + log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone.getId())); + controller().applications().deploy(application, zone.getId(), target); } } @Override - protected boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { - return minVersion(zone, application, Node::currentVersion).map(target::equals) - .orElse(true) - && application.configConvergedIn(zone, controller()); + protected boolean convergedOn(Version target, SystemApplication application, ZoneApi zone) { + Optional<Version> minVersion = minVersion(zone, application, Node::currentVersion); + // Skip application convergence check if there are no nodes belonging to the application in the zone + if (minVersion.isEmpty()) return true; + + return minVersion.get().equals(target) + && application.configConvergedIn(zone.getId(), controller(), Optional.of(target)); } @Override - protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone) { + protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { return eligibleForUpgrade(node); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 5c4adb46f6b..74cea37f3a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -16,12 +16,14 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -30,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -39,6 +42,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -46,6 +51,7 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.TreeMap; +import java.util.stream.Collectors; /** * Serializes {@link Application} to/from slime. @@ -55,6 +61,13 @@ import java.util.TreeMap; */ public class ApplicationSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + // Application fields private final String idField = "id"; private final String createdAtField = "createdAt"; @@ -71,8 +84,14 @@ public class ApplicationSerializer { private final String writeQualityField = "writeQuality"; private final String queryQualityField = "queryQuality"; private final String pemDeployKeyField = "pemDeployKey"; - private final String rotationField = "rotation"; + private final String assignedRotationsField = "assignedRotations"; + private final String assignedRotationEndpointField = "endpointId"; + private final String assignedRotationClusterField = "clusterId"; + private final String assignedRotationRotationField = "rotationId"; + private final String rotationsField = "endpoints"; + private final String deprecatedRotationField = "rotation"; private final String rotationStatusField = "rotationStatus"; + private final String applicationCertificateField = "applicationCertificate"; // Deployment fields private final String zoneField = "zone"; @@ -164,8 +183,11 @@ public class ApplicationSerializer { root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); application.pemDeployKey().ifPresent(pemDeployKey -> root.setString(pemDeployKeyField, pemDeployKey)); - application.rotation().ifPresent(rotation -> root.setString(rotationField, rotation.asString())); + application.legacyRotation().ifPresent(rotation -> root.setString(deprecatedRotationField, rotation.asString())); + rotationsToSlime(application.assignedRotations(), root, rotationsField); + assignedRotationsToSlime(application.assignedRotations(), root, assignedRotationsField); toSlime(application.rotationStatus(), root.setArray(rotationStatusField)); + application.applicationCertificate().ifPresent(cert -> root.setString(applicationCertificateField, cert.secretsKeyNamePrefix())); return slime; } @@ -336,6 +358,21 @@ public class ApplicationSerializer { return clusterMetricsList; } + private void rotationsToSlime(List<AssignedRotation> rotations, Cursor parent, String fieldName) { + final var rotationsArray = parent.setArray(fieldName); + rotations.forEach(rot -> rotationsArray.addString(rot.rotationId().asString())); + } + + private void assignedRotationsToSlime(List<AssignedRotation> rotations, Cursor parent, String fieldName) { + final var rotationsArray = parent.setArray(fieldName); + for (var rotation : rotations) { + final var object = rotationsArray.addObject(); + object.setString(assignedRotationEndpointField, rotation.endpointId().id()); + object.setString(assignedRotationRotationField, rotation.rotationId().asString()); + object.setString(assignedRotationClusterField, rotation.clusterId().value()); + } + } + // ------------------ Deserialization public Application fromSlime(Slime slime) { @@ -355,12 +392,13 @@ public class ApplicationSerializer { ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble()); Optional<String> pemDeployKey = optionalString(root.field(pemDeployKeyField)); - Optional<RotationId> rotation = rotationFromSlime(root.field(rotationField)); + List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(deploymentSpec, root); Map<HostName, RotationStatus> rotationStatus = rotationStatusFromSlime(root.field(rotationStatusField)); + Optional<ApplicationCertificate> applicationCertificate = optionalString(root.field(applicationCertificateField)).map(ApplicationCertificate::new); return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, rotation, rotationStatus); + pemDeployKey, assignedRotations, rotationStatus, applicationCertificate); } private List<Deployment> deploymentsFromSlime(Inspector array) { @@ -541,7 +579,57 @@ public class ApplicationSerializer { Instant.ofEpochMilli(object.field(atField).asLong()))); } - private Optional<RotationId> rotationFromSlime(Inspector field) { + private List<AssignedRotation> assignedRotationsFromSlime(DeploymentSpec deploymentSpec, Inspector root) { + final var assignedRotations = new LinkedHashMap<EndpointId, AssignedRotation>(); + + // Add the legacy rotation field to the set - this needs to be first + // TODO: Remove when we retire the rotations field + final var legacyRotation = legacyRotationFromSlime(root.field(deprecatedRotationField)); + if (legacyRotation.isPresent() && deploymentSpec.globalServiceId().isPresent()) { + final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); + final var regions = deploymentSpec.zones().stream().flatMap(zone -> zone.region().stream()).collect(Collectors.toSet()); + assignedRotations.putIfAbsent(EndpointId.default_(), new AssignedRotation(clusterId, EndpointId.default_(), legacyRotation.get(), regions)); + } + + // Now add the same entries from "stupid" list of rotations + // TODO: Remove when we retire the rotations field + final var rotations = rotationListFromSlime(root.field(rotationsField)); + for (var rotation : rotations) { + final var regions = deploymentSpec.zones().stream().flatMap(zone -> zone.region().stream()).collect(Collectors.toSet()); + if (deploymentSpec.globalServiceId().isPresent()) { + final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); + assignedRotations.putIfAbsent(EndpointId.default_(), new AssignedRotation(clusterId, EndpointId.default_(), rotation, regions)); + } + } + + // Last - add the actual entries we want. Do _not_ remove this during clean-up + root.field(assignedRotationsField).traverse((ArrayTraverser) (idx, inspector) -> { + final var clusterId = new ClusterSpec.Id(inspector.field(assignedRotationClusterField).asString()); + final var endpointId = EndpointId.of(inspector.field(assignedRotationEndpointField).asString()); + final var rotationId = new RotationId(inspector.field(assignedRotationRotationField).asString()); + final var regions = deploymentSpec.endpoints().stream() + .filter(endpoint -> endpoint.endpointId().equals(endpointId.id())) + .flatMap(endpoint -> endpoint.regions().stream()) + .collect(Collectors.toSet()); + assignedRotations.putIfAbsent(endpointId, new AssignedRotation(clusterId, endpointId, rotationId, regions)); + }); + + return List.copyOf(assignedRotations.values()); + } + + private List<RotationId> rotationListFromSlime(Inspector field) { + final var rotations = new ArrayList<RotationId>(); + + field.traverse((ArrayTraverser) (idx, inspector) -> { + final var rotation = new RotationId(inspector.asString()); + rotations.add(rotation); + }); + + return rotations; + } + + // TODO: Remove after June 2019 once the 'rotation' field is gone from storage + private Optional<RotationId> legacyRotationFromSlime(Inspector field) { return field.valid() ? optionalString(field).map(RotationId::new) : Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java index 5bcb155efcb..d18e561ce5d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java @@ -19,6 +19,13 @@ import java.util.function.Function; */ public class AuditLogSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String entriesField = "entries"; private static final String atField = "at"; private static final String principalField = "principal"; @@ -60,6 +67,7 @@ public class AuditLogSerializer { switch (method) { case POST: return "POST"; case PATCH: return "PATCH"; + case PUT: return "PUT"; case DELETE: return "DELETE"; default: throw new IllegalArgumentException("No serialization defined for method " + method); } @@ -69,6 +77,7 @@ public class AuditLogSerializer { switch (field.asString()) { case "POST": return AuditLog.Entry.Method.POST; case "PATCH": return AuditLog.Entry.Method.PATCH; + case "PUT": return AuditLog.Entry.Method.PUT; case "DELETE": return AuditLog.Entry.Method.DELETE; default: throw new IllegalArgumentException("Unknown serialized value '" + field.asString() + "'"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java index a87875da104..2cb981aac03 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java @@ -18,6 +18,13 @@ import java.util.Map; */ public class ConfidenceOverrideSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private final static String overridesField = "overrides"; public Slime toSlime(Map<Version, Confidence> overrides) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index a2e5a0c78f7..d704d701cf0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -106,6 +106,10 @@ public class CuratorDb { CuratorDb(Curator curator, Duration tryLockTimeout) { this.curator = curator; this.tryLockTimeout = tryLockTimeout; + + // TODO: Remove after 7.60 + curator.delete(root.append("openStackServerPool")); + curator.delete(root.append("vespaServerPool")); } /** Returns all hosts configured to be part of this ZooKeeper cluster */ @@ -168,16 +172,6 @@ public class CuratorDb { return lock(lockPath(provisionStateId), Duration.ofSeconds(1)); } - @SuppressWarnings("unused") // Called by internal code - public Lock lockVespaServerPool() { - return lock(lockRoot.append("vespaServerPoolLock"), Duration.ofSeconds(1)); - } - - @SuppressWarnings("unused") // Called by internal code - public Lock lockOpenStackServerPool() { - return lock(lockRoot.append("openStackServerPoolLock"), Duration.ofSeconds(1)); - } - public Lock lockOsVersions() { return lock(lockRoot.append("osTargetVersion"), defaultLockTimeout); } @@ -469,26 +463,6 @@ public class CuratorDb { return curator.getChildren(provisionStatePath()); } - @SuppressWarnings("unused") - public Optional<byte[]> readVespaServerPool() { - return curator.getData(vespaServerPoolPath()); - } - - @SuppressWarnings("unused") - public void writeVespaServerPool(byte[] data) { - curator.set(vespaServerPoolPath(), data); - } - - @SuppressWarnings("unused") - public Optional<byte[]> readOpenStackServerPool() { - return curator.getData(openStackServerPoolPath()); - } - - @SuppressWarnings("unused") - public void writeOpenStackServerPool(byte[] data) { - curator.set(openStackServerPoolPath(), data); - } - // -------------- Routing policies ---------------------------------------- public void writeRoutingPolicies(ApplicationId application, Set<RoutingPolicy> policies) { @@ -589,14 +563,6 @@ public class CuratorDb { return provisionStatePath().append(provisionId); } - private static Path vespaServerPoolPath() { - return root.append("vespaServerPool"); - } - - private static Path openStackServerPoolPath() { - return root.append("openStackServerPool"); - } - private static Path tenantPath(TenantName name) { return tenantRoot.append(name.value()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java index 40781ac6e92..418038d4f1e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java @@ -27,6 +27,13 @@ import java.util.stream.Collectors; */ class LogSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String idField = "id"; private static final String typeField = "type"; private static final String timestampField = "at"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java index 3dfb5ffe5f8..e3dedd65e68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java @@ -24,6 +24,13 @@ import java.util.ArrayList; */ public class NameServiceQueueSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String requestsField = "requests"; private static final String requestType = "requestType"; private static final String recordsField = "records"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java index 21f8b1bcb80..d68e24a27ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java @@ -20,6 +20,13 @@ import java.util.TreeSet; */ public class OsVersionSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String versionsField = "versions"; private static final String versionField = "version"; private static final String cloudField = "cloud"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java index 3e3c0df1673..88805f54d65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java @@ -26,6 +26,13 @@ import java.util.TreeMap; */ public class OsVersionStatusSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String versionsField = "versions"; private static final String versionField = "version"; private static final String nodesField = "nodes"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index a9c6c54a44a..9cfce8dc16a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -24,6 +24,13 @@ import java.util.function.Function; */ public class RoutingPolicySerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String routingPoliciesField = "routingPolicies"; private static final String clusterField = "cluster"; private static final String canonicalNameField = "canonicalName"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index f29af1055d0..1c95c9766f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -56,6 +56,13 @@ import static java.util.Comparator.comparing; */ class RunSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String stepsField = "steps"; private static final String applicationField = "id"; private static final String jobTypeField = "type"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 56e80068908..3a4e6c3954c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -29,6 +29,13 @@ import java.util.Optional; */ public class TenantSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String nameField = "name"; private static final String typeField = "type"; private static final String athenzDomainField = "athenzDomain"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java index 5edae803fdb..e5897963254 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java @@ -13,6 +13,13 @@ import com.yahoo.slime.Slime; */ public class VersionSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String versionField = "version"; public Slime toSlime(Version version) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 72d38bbee5f..207a5f8dcf9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -27,6 +27,13 @@ import java.util.Set; */ public class VersionStatusSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + // VersionStatus fields private static final String versionsField = "versions"; @@ -35,6 +42,7 @@ public class VersionStatusSerializer { private static final String committedAtField = "releasedAt"; private static final String isControllerVersionField = "isCurrentControllerVersion"; private static final String isSystemVersionField = "isCurrentSystemVersion"; + private static final String isReleasedField = "isReleased"; private static final String deploymentStatisticsField = "deploymentStatistics"; private static final String confidenceField = "confidence"; private static final String configServersField = "configServerHostnames"; @@ -66,6 +74,7 @@ public class VersionStatusSerializer { object.setLong(committedAtField, version.committedAt().toEpochMilli()); object.setBool(isControllerVersionField, version.isControllerVersion()); object.setBool(isSystemVersionField, version.isSystemVersion()); + object.setBool(isReleasedField, version.isReleased()); deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField)); object.setString(confidenceField, version.confidence().name()); configServersToSlime(version.systemApplicationHostnames(), object.setArray(configServersField)); @@ -98,6 +107,7 @@ public class VersionStatusSerializer { Instant.ofEpochMilli(object.field(committedAtField).asLong()), object.field(isControllerVersionField).asBool(), object.field(isSystemVersionField).asBool(), + object.field(isReleasedField).valid() ? object.field(isReleasedField).asBool() : true, configServersFromSlime(object.field(configServersField)), VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) ); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 01d9a01a316..73a029ad3b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -5,14 +5,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import org.apache.http.Header; import org.apache.http.client.config.RequestConfig; @@ -114,9 +115,9 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { if ( ! environmentName.isEmpty()) zones = zones.in(Environment.from(environmentName)); - for (ZoneId zoneId : zones.ids()) { + for (ZoneApi zone : zones.zones()) { responseStructure.uris.add(proxyRequest.getScheme() + "://" + proxyRequest.getControllerPrefix() + - zoneId.environment().name() + "/" + zoneId.region().value()); + zone.getEnvironment().value() + "/" + zone.getRegionName().value()); } JsonNode node = mapper.valueToTree(responseStructure); return new ProxyResponse(proxyRequest, node.toString(), 200, Optional.empty(), "application/json"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d32b3f009f4..868272e5051 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -7,15 +7,13 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -31,9 +29,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.application.v4.ApplicationResource; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import com.yahoo.vespa.hosted.controller.api.application.v4.TenantResource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; @@ -50,7 +46,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; @@ -66,11 +61,11 @@ import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; +import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -95,6 +90,8 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Base64; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -102,6 +99,7 @@ import java.util.Scanner; import java.util.Set; import java.util.StringJoiner; import java.util.logging.Level; +import java.util.stream.Collectors; import static java.util.stream.Collectors.joining; @@ -174,15 +172,29 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/user")) return authenticatedUser(request); if (path.matches("/application/v4/tenant")) return tenants(request); if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), Optional.empty(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/metering")) return metering(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance")) return applications(path.get("tenant"), Optional.of(path.get("application")), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return application(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -195,53 +207,65 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handlePUT(Path path, HttpRequest request) { if (path.matches("/application/v4/user")) return createUser(request); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) - return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); return ErrorResponse.notFoundError("Nothing at " + path); } private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), false, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return submit(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"), path.get("instance"), request); return ErrorResponse.notFoundError("Nothing at " + path); } private HttpResponse handlePATCH(Path path, HttpRequest request) { - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) - return patchApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return patchApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return patchApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); return ErrorResponse.notFoundError("Nothing at " + path); } private HttpResponse handleDELETE(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "all"); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("choice")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", "all"); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application"), "default"); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return deleteApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all"); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application"), path.get("instance")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) - return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); return ErrorResponse.notFoundError("Nothing at " + path); } private HttpResponse handleOPTIONS() { // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother // spelling out the methods supported at each path, which we should - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); return response; } @@ -299,25 +323,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse applications(String tenantName, HttpRequest request) { + private HttpResponse applications(String tenantName, Optional<String> applicationName, HttpRequest request) { TenantName tenant = TenantName.from(tenantName); Slime slime = new Slime(); Cursor array = slime.setArray(); - for (Application application : controller.applications().asList(tenant)) + for (Application application : controller.applications().asList(tenant)) { + if (applicationName.isPresent() && ! application.id().application().toString().equals(applicationName.get())) + continue; toSlime(application, array.addObject(), request); + } return new SlimeJsonResponse(slime); } - private HttpResponse application(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse application(String tenantName, String applicationName, String instanceName, HttpRequest request) { Slime slime = new Slime(); - toSlime(slime.setObject(), getApplication(tenantName, applicationName), request); + toSlime(slime.setObject(), getApplication(tenantName, applicationName, instanceName), request); return new SlimeJsonResponse(slime); } - private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse patchApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); StringJoiner messageBuilder = new StringJoiner("\n").setEmptyValue("No applicable changes."); - controller.applications().lockOrThrow(ApplicationId.from(tenantName, applicationName, "default"), application -> { + controller.applications().lockOrThrow(ApplicationId.from(tenantName, applicationName, instanceName), application -> { Inspector majorVersionField = requestObject.field("majorVersion"); if (majorVersionField.valid()) { Integer majorVersion = majorVersionField.asLong() == 0 ? null : (int) majorVersionField.asLong(); @@ -337,8 +364,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new MessageResponse(messageBuilder.toString()); } - private Application getApplication(String tenantName, String applicationName) { - ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); + private Application getApplication(String tenantName, String applicationName, String instanceName) { + ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); return controller.applications().get(applicationId) .orElseThrow(() -> new NotExistsException(applicationId + " not found")); } @@ -482,7 +509,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { }); // Compile version. The version that should be used when building an application - object.setString("compileVersion", controller.applications().oldestInstalledPlatform(application.id()).toFullString()); + object.setString("compileVersion", compileVersion(application.id()).toFullString()); application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); @@ -496,10 +523,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .map(URI::toString) .forEach(globalRotationsArray::addString); - application.rotation().ifPresent(rotation -> object.setString("rotationId", rotation.asString())); + + application.rotations().stream().findFirst().ifPresent(rotation -> object.setString("rotationId", rotation.asString())); // Per-cluster rotations - Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies(application.id()); + Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(application.id()); for (RoutingPolicy policy : routingPolicies) { policy.rotationEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) @@ -515,7 +543,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { for (Deployment deployment : deployments) { Cursor deploymentObject = instancesArray.addObject(); - if (application.rotation().isPresent() && deployment.zone().environment() == Environment.prod) { + if ((! application.rotations().isEmpty()) && deployment.zone().environment() == Environment.prod) { toSlime(application.rotationStatus(deployment), deploymentObject); } @@ -528,7 +556,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { deploymentObject.setString("url", withPath(request.getUri().getPath() + "/environment/" + deployment.zone().environment().value() + "/region/" + deployment.zone().region().value() + - "/instance/" + application.id().instance().value(), + ( request.getUri().getPath().contains("/instance/") ? "" : "/instance/" + application.id().instance().value()), request.getUri()).toString()); } } @@ -585,7 +613,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add endpoint(s) defined by routing policies var endpointArray = response.setArray("endpoints"); - for (var policy : controller.applications().routingPolicies(deploymentId.applicationId())) { + for (var policy : controller.applications().routingPolicies().get(deploymentId)) { Cursor endpointObject = endpointArray.addObject(); Endpoint endpoint = policy.endpointIn(controller.system()); endpointObject.setString("cluster", policy.cluster().value()); @@ -598,7 +626,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // ask the routing layer here Cursor serviceUrlArray = response.setArray("serviceUrls"); controller.applications().getDeploymentEndpoints(deploymentId) - .ifPresent(endpoints -> endpoints.forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()))); + .forEach(endpoint -> serviceUrlArray.addString(endpoint.toString())); response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); @@ -659,6 +687,30 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return controller.zoneRegistry().getMonitoringSystemUri(deploymentId); } + /** + * Returns a non-broken, released version at least as old as the oldest platform the given application is on. + * + * If no known version is applicable, the newest version at least as old as the oldest platform is selected, + * among all versions released for this system. If no such versions exists, throws an IllegalStateException. + */ + private Version compileVersion(ApplicationId id) { + Version oldestPlatform = controller.applications().oldestInstalledPlatform(id); + return controller.versionStatus().versions().stream() + .filter(version -> version.confidence().equalOrHigherThan(VespaVersion.Confidence.low)) + .filter(VespaVersion::isReleased) + .map(VespaVersion::versionNumber) + .filter(version -> ! version.isAfter(oldestPlatform)) + .max(Comparator.naturalOrder()) + .orElseGet(() -> controller.mavenRepository().metadata().versions().stream() + .filter(version -> ! version.isAfter(oldestPlatform)) + .filter(version -> ! controller.versionStatus().versions().stream() + .map(VespaVersion::versionNumber) + .collect(Collectors.toSet()).contains(version)) + .max(Comparator.naturalOrder()) + .orElseThrow(() -> new IllegalStateException("No available releases of " + + controller.mavenRepository().artifactId()))); + } + private HttpResponse setGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region, boolean inService, HttpRequest request) { Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); ZoneId zone = ZoneId.from(environment, region); @@ -707,7 +759,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); Application application = controller.applications().require(applicationId); ZoneId zone = ZoneId.from(environment, region); - if (!application.rotation().isPresent()) { + if (application.rotations().isEmpty()) { throw new NotExistsException("global rotation does not exist for " + application); } Deployment deployment = application.deployments().get(zone); @@ -721,11 +773,50 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse deploying(String tenant, String application, HttpRequest request) { - Application app = controller.applications().require(ApplicationId.from(tenant, application, "default")); + private HttpResponse metering(String tenant, String application, HttpRequest request) { Slime slime = new Slime(); Cursor root = slime.setObject(); - if (!app.change().isEmpty()) { + + Cursor currentRate = root.setObject("currentrate"); + currentRate.setDouble("cpu", 0); + currentRate.setDouble("mem", 0); + currentRate.setDouble("disk", 0); + + Cursor thismonth = root.setObject("thismonth"); + thismonth.setDouble("cpu", 0); + thismonth.setDouble("mem", 0); + thismonth.setDouble("disk", 0); + + Cursor lastmonth = root.setObject("lastmonth"); + lastmonth.setDouble("cpu", 0); + lastmonth.setDouble("mem", 0); + lastmonth.setDouble("disk", 0); + + Cursor details = root.setObject("details"); + + Cursor detailsCpu = details.setObject("cpu"); + Cursor detailsCpuDummyApp = detailsCpu.setObject("dummy"); + Cursor detailsCpuDummyData = detailsCpuDummyApp.setArray("data"); + + // The data array should be filled with objects like: { unixms: <number>, valur: <number } + + Cursor detailsMem = details.setObject("mem"); + Cursor detailsMemDummyApp = detailsMem.setObject("dummy"); + Cursor detailsMemDummyData = detailsMemDummyApp.setArray("data"); + + Cursor detailsDisk = details.setObject("disk"); + Cursor detailsDiskDummyApp = detailsDisk.setObject("dummy"); + Cursor detailsDiskDummyData = detailsDiskDummyApp.setArray("data"); + + + return new SlimeJsonResponse(slime); + } + + private HttpResponse deploying(String tenant, String application, String instance, HttpRequest request) { + Application app = controller.applications().require(ApplicationId.from(tenant, application, instance)); + Slime slime = new Slime(); + Cursor root = slime.setObject(); + if ( ! app.change().isEmpty()) { app.change().platform().ifPresent(version -> root.setString("platform", version.toString())); app.change().application().ifPresent(applicationVersion -> root.setString("application", applicationVersion.id())); root.setBool("pinned", app.change().isPinned()); @@ -800,9 +891,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return tenant(controller.tenants().require(TenantName.from(tenantName)), request); } - private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse createApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); try { Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user ? Optional.empty() @@ -822,10 +913,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** Trigger deployment of the given Vespa version if a valid one is given, e.g., "7.8.9". */ - private HttpResponse deployPlatform(String tenantName, String applicationName, boolean pin, HttpRequest request) { + private HttpResponse deployPlatform(String tenantName, String applicationName, String instanceName, boolean pin, HttpRequest request) { request = controller.auditLogger().log(request); String versionString = readToString(request.getData()); - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Version version = Version.fromString(versionString); @@ -833,12 +924,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { version = controller.systemVersion(); if ( ! systemHasVersion(version)) throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " + - "Version is not active in this system. " + - "Active versions: " + controller.versionStatus().versions() - .stream() - .map(VespaVersion::versionNumber) - .map(Version::toString) - .collect(joining(", "))); + "Version is not active in this system. " + + "Active versions: " + controller.versionStatus().versions() + .stream() + .map(VespaVersion::versionNumber) + .map(Version::toString) + .collect(joining(", "))); Change change = Change.of(version); if (pin) change = change.withPin(); @@ -850,9 +941,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** Trigger deployment to the last known application package for the given application. */ - private HttpResponse deployApplication(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse deployApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { controller.auditLogger().log(request); - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Change change = Change.of(application.get().deploymentJobs().statusOf(JobType.component).get().lastSuccess().get().application()); @@ -863,8 +954,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** Cancel ongoing change for given application, e.g., everything with {"cancel":"all"} */ - private HttpResponse cancelDeploy(String tenantName, String applicationName, String choice) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + private HttpResponse cancelDeploy(String tenantName, String applicationName, String instanceName, String choice) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Change change = application.get().change(); @@ -891,12 +982,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Optional<Hostname> hostname = Optional.ofNullable(request.getProperty("hostname")).map(Hostname::new); controller.applications().restart(deploymentId, hostname); - // TODO: Change to return JSON - return new StringResponse("Requested restart of " + path(TenantResource.API_PATH, tenantName, - ApplicationResource.API_PATH, applicationName, - EnvironmentResource.API_PATH, environment, - "region", region, - "instance", instanceName)); + return new MessageResponse("Requested restart of " + deploymentId); } private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) { @@ -919,7 +1005,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Slime slime = new Slime(); Cursor rootObject = slime.setObject(); rootObject.setString("message", "Deployment started in " + runId); - rootObject.setString("location", controller.zoneRegistry().dashboardUrl(runId).toString()); + rootObject.setLong("run", runId.number()); return new SlimeJsonResponse(slime); } @@ -934,11 +1020,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); /* - * Special handling of the zone application (the only system application with an application package) + * Special handling of the proxy application (the only system application with an application package) * Setting any other deployOptions here is not supported for now (e.g. specifying version), but * this might be handy later to handle emergency downgrades. */ - boolean isZoneApplication = SystemApplication.zone.id().equals(applicationId); + boolean isZoneApplication = SystemApplication.proxy.id().equals(applicationId); if (isZoneApplication) { // TODO jvenstad: Separate out. // Make it explicit that version is not yet supported here String versionStr = deployOptions.field("vespaVersion").asString(); @@ -956,7 +1042,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { throw new IllegalArgumentException("Deployment of system applications is not permitted until system version is determined"); } ActivateResult result = controller.applications() - .deploySystemApplicationPackage(SystemApplication.zone, zone, systemVersion.get().versionNumber()); + .deploySystemApplicationPackage(SystemApplication.proxy, zone, systemVersion.get().versionNumber()); return new SlimeJsonResponse(toSlime(result)); } @@ -1037,59 +1123,23 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return tenant(tenant.get(), request); } - private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + private HttpResponse deleteApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); Optional<Credentials> credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user ? Optional.empty() : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteApplication(id, credentials); - return new EmptyJsonResponse(); // TODO: Replicates current behavior but should return a message response instead + return new MessageResponse("Deleted application " + id); } private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); // Attempt to deactivate application even if the deployment is not known by the controller - controller.applications().deactivate(application.id(), ZoneId.from(environment, region)); + DeploymentId deploymentId = new DeploymentId(application.id(), ZoneId.from(environment, region)); + controller.applications().deactivate(deploymentId.applicationId(), deploymentId.zoneId()); - // TODO: Change to return JSON - return new StringResponse("Deactivated " + path(TenantResource.API_PATH, tenantName, - ApplicationResource.API_PATH, applicationName, - EnvironmentResource.API_PATH, environment, - "region", region, - "instance", instanceName)); - } - - /** - * Promote application Chef environments. To be used by component jobs only - */ - private HttpResponse promoteApplication(String tenantName, String applicationName, HttpRequest request) { - try{ - ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); - String sourceEnvironment = chefEnvironment.systemChefEnvironment(); - String targetEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); - controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); - return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); - } catch (Exception e) { - log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s)", tenantName, applicationName), e); - return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); - } - } - - /** - * Promote application Chef environments for jobs that deploy applications - */ - private HttpResponse promoteApplicationDeployment(String tenantName, String applicationName, String environmentName, String regionName, String instanceName, HttpRequest request) { - try { - ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); - String sourceEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); - String targetEnvironment = chefEnvironment.applicationTargetEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName), Environment.from(environmentName), RegionName.from(regionName)); - controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); - return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); - } catch (Exception e) { - log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s %s.%s)", tenantName, applicationName, environmentName, regionName), e); - return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); - } + return new MessageResponse("Deactivated " + deploymentId); } private HttpResponse notifyJobCompletion(String tenant, String application, HttpRequest request) { @@ -1108,6 +1158,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } } + private HttpResponse testConfig(ApplicationId id, JobType type) { + var endpoints = controller.applications().clusterEndpoints(id, controller.jobController().testedZoneAndProductionZones(id, type)); + return new SlimeJsonResponse(new TestConfigSerializer(controller.system()).configSlime(id, + type, + endpoints, + Collections.emptyMap())); + } + private static DeploymentJobs.JobReport toJobReport(String tenantName, String applicationName, Inspector report) { Optional<DeploymentJobs.JobError> jobError = Optional.empty(); if (report.field("jobError").valid()) { @@ -1144,7 +1202,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void toSlime(Cursor object, Tenant tenant, HttpRequest request) { object.setString("tenant", tenant.name().value()); - object.setString("type", tentantType(tenant)); + object.setString("type", tenantType(tenant)); switch (tenant.type()) { case athenz: AthenzTenant athenzTenant = (AthenzTenant) tenant; @@ -1168,7 +1226,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } Cursor applicationArray = object.setArray("applications"); for (Application application : controller.applications().asList(tenant.name())) { - if (application.id().instance().isDefault()) {// TODO: Skip non-default applications until supported properly + if ( ! application.id().instance().isTester()) { if (recurseOverApplications(request)) toSlime(applicationArray.addObject(), application, request); else @@ -1181,7 +1239,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void tenantInTenantsListToSlime(Tenant tenant, URI requestURI, Cursor object) { object.setString("tenant", tenant.name().value()); Cursor metaData = object.setObject("metaData"); - metaData.setString("type", tentantType(tenant)); + metaData.setString("type", tenantType(tenant)); switch (tenant.type()) { case athenz: AthenzTenant athenzTenant = (AthenzTenant) tenant; @@ -1257,8 +1315,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { object.setString("tenant", application.id().tenant().value()); object.setString("application", application.id().application().value()); object.setString("instance", application.id().instance().value()); - object.setString("url", withPath("/application/v4/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value(), request.getUri()).toString()); + object.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value() + + "/instance/" + application.id().instance().value(), + request.getUri()).toString()); } private Slime toSlime(ActivateResult result) { @@ -1389,7 +1450,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ImmutableSet.of("all", "true", "deployment").contains(request.getProperty("recursive")); } - private static String tentantType(Tenant tenant) { + private static String tenantType(Tenant tenant) { switch (tenant.type()) { case user: return "USER"; case athenz: return "ATHENS"; @@ -1411,7 +1472,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new RunId(appIdFromPath(path), jobTypeFromPath(path), number); } - private HttpResponse submit(String tenant, String application, HttpRequest request) { + private HttpResponse submit(String tenant, String application, String instance, HttpRequest request) { Map<String, byte[]> dataParts = parseDataParts(request); Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); SourceRevision sourceRevision = toSourceRevision(submitOptions); @@ -1426,6 +1487,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, + instance, sourceRevision, authorEmail, projectId, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java deleted file mode 100644 index 7c32e48e218..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; - -/** - * Represents Chef environments for applications/deployments. Used for promotion of Chef environments - * - * @author mortent - */ -public class ApplicationChefEnvironment { - - private final String systemChefEnvironment; - private final String systemSuffix; - - public ApplicationChefEnvironment(SystemName system) { - if (system == SystemName.main) { - systemChefEnvironment = "hosted-verified-prod"; - systemSuffix = ""; - } else { - systemChefEnvironment = "hosted-infra-cd"; - systemSuffix = "-cd"; - } - } - - public String systemChefEnvironment() { - return systemChefEnvironment; - } - - public String applicationSourceEnvironment(TenantName tenantName, ApplicationName applicationName) { - // placeholder and component already used in legacy chef promotion - return String.format("hosted-instance%s_%s_%s_placeholder_component_default", systemSuffix, tenantName, applicationName); - } - - public String applicationTargetEnvironment(TenantName tenantName, ApplicationName applicationName, Environment environment, RegionName regionName) { - return String.format("hosted-instance%s_%s_%s_%s_%s_default", systemSuffix, tenantName, applicationName, regionName, environment); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java index be3222cc1a8..e343615f066 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java @@ -8,16 +8,13 @@ import java.io.OutputStream; /** * @author bratseth */ -public class EmptyJsonResponse extends HttpResponse { +public class EmptyResponse extends HttpResponse { - public EmptyJsonResponse() { + public EmptyResponse() { super(200); } @Override public void render(OutputStream stream) {} - @Override - public String getContentType() { return "application/json"; } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 48cf2a7824d..b34ea79c670 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -6,26 +6,26 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.JobController; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; @@ -95,11 +95,13 @@ class JobControllerApiHandlerHelper { Cursor responseObject = slime.setObject(); Cursor lastVersionsObject = responseObject.setObject("lastVersions"); - lastPlatformToSlime(lastVersionsObject.setObject("platform"), controller, application, change, steps); - lastApplicationToSlime(lastVersionsObject.setObject("application"), application, change, steps, controller); + if (application.deploymentJobs().statusOf(component).flatMap(JobStatus::lastSuccess).isPresent()) { + lastPlatformToSlime(lastVersionsObject.setObject("platform"), controller, application, change, steps); + lastApplicationToSlime(lastVersionsObject.setObject("application"), application, change, steps, controller); + } + Cursor deployingObject = responseObject.setObject("deploying"); if ( ! change.isEmpty()) { - Cursor deployingObject = responseObject.setObject("deploying"); change.platform().ifPresent(version -> deployingObject.setString("platform", version.toString())); change.application().ifPresent(version -> applicationVersionToSlime(deployingObject.setObject("application"), version)); } @@ -132,6 +134,17 @@ class JobControllerApiHandlerHelper { running, baseUriForJobs.resolve(baseUriForJobs.getPath() + "/" + type.jobName()).normalize()); }); + + Cursor devJobsObject = responseObject.setObject("devJobs"); + for (JobType type : JobType.allIn(controller.system())) + if ( type.environment() != null + && type.environment().isManuallyDeployed() + && application.deployments().containsKey(type.zone(controller.system()))) + controller.jobController().last(application.id(), type) + .ifPresent(last -> runToSlime(devJobsObject.setObject(type.jobName()).setArray("runs").addObject(), + last, + baseUriForJobs.resolve(baseUriForJobs.getPath() + "/" + type.jobName()).normalize())); + return new SlimeJsonResponse(slime); } @@ -327,9 +340,10 @@ class JobControllerApiHandlerHelper { /** Returns the status of the task represented by the given step, if it has started. */ private static Optional<String> taskStatus(Step step, Run run) { - return run.readySteps().contains(step) ? Optional.of("running") - : run.steps().get(step) != unfinished ? Optional.of(run.steps().get(step).name()) - : Optional.empty(); + return run.readySteps().contains(step) ? Optional.of("running") + : Optional.ofNullable(run.steps().get(step)) + .filter(status -> status != unfinished) + .map(Step.Status::name); } /** Returns a response with the runs for the given job type. */ @@ -351,6 +365,9 @@ class JobControllerApiHandlerHelper { private static void applicationVersionToSlime(Cursor versionObject, ApplicationVersion version) { versionObject.setString("hash", version.id()); + if (version.isUnknown()) + return; + versionObject.setLong("build", version.buildNumber().getAsLong()); Cursor sourceObject = versionObject.setObject("source"); sourceObject.setString("gitRepository", version.source().get().repository()); @@ -399,10 +416,10 @@ class JobControllerApiHandlerHelper { * * @return Response with the new application version */ - static HttpResponse submitResponse(JobController jobController, String tenant, String application, + static HttpResponse submitResponse(JobController jobController, String tenant, String application, String instance, SourceRevision sourceRevision, String authorEmail, long projectId, ApplicationPackage applicationPackage, byte[] testPackage) { - ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, "default"), + ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, instance), sourceRevision, authorEmail, projectId, @@ -427,8 +444,8 @@ class JobControllerApiHandlerHelper { } /** Unregisters the application from the internal deployment pipeline. */ - static HttpResponse unregisterResponse(JobController jobs, String tenantName, String applicationName) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + static HttpResponse unregisterResponse(JobController jobs, String tenantName, String applicationName, String instanceName) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); jobs.unregister(id); Slime slime = new Slime(); slime.setObject().setString("message", "Unregistered '" + id + "' from internal deployment pipeline."); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index a59e0e9130f..dde79e78850 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -100,7 +100,8 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { } private HttpResponse setActive(String jobName, boolean active) { - if ( ! maintenance.jobControl().jobs().contains(jobName)) + boolean activatingInactiveJob = active && !maintenance.jobControl().isActive(jobName); + if (!activatingInactiveJob && !maintenance.jobControl().jobs().contains(jobName)) return ErrorResponse.notFoundError("No job named '" + jobName + "'"); maintenance.jobControl().setActive(jobName, active); return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java index bae790a49ad..796f786d823 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java @@ -6,7 +6,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.restapi.Path; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; @@ -19,10 +19,10 @@ import static com.yahoo.jdisc.http.HttpRequest.Method.GET; public class CostApiHandler extends LoggingRequestHandler { private final Controller controller; - private final NodeRepositoryClientInterface nodeRepository; + private final NodeRepository nodeRepository; private final SelfHostedCostConfig selfHostedCostConfig; - public CostApiHandler(Context ctx, Controller controller, NodeRepositoryClientInterface nodeRepository, SelfHostedCostConfig selfHostedCostConfig) { + public CostApiHandler(Context ctx, Controller controller, NodeRepository nodeRepository, SelfHostedCostConfig selfHostedCostConfig) { super(ctx); this.controller = controller; this.nodeRepository = nodeRepository; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java index 18c00d69b62..919cade1b05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java @@ -4,8 +4,8 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; @@ -25,7 +25,7 @@ public class CostCalculator { private static final double SELF_HOSTED_DISCOUNT = .5; - public static String resourceShareByPropertyToCsv(NodeRepositoryClientInterface nodeRepository, + public static String resourceShareByPropertyToCsv(NodeRepository nodeRepository, Controller controller, Clock clock, SelfHostedCostConfig selfHostedCostConfig, @@ -34,8 +34,8 @@ public class CostCalculator { String date = LocalDate.now(clock).toString(); List<NodeRepositoryNode> nodes = controller.zoneRegistry().zones() - .reachable().in(Environment.prod).ofCloud(cloudName).ids().stream() - .flatMap(zoneId -> uncheck(() -> nodeRepository.listNodes(zoneId, true).nodes().stream())) + .reachable().in(Environment.prod).ofCloud(cloudName).zones().stream() + .flatMap(zone -> uncheck(() -> nodeRepository.listNodes(zone.getId()).nodes().stream())) .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) .collect(Collectors.toList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 978b7e4397d..44b67a186b8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -8,18 +8,18 @@ import com.yahoo.config.provision.HostName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.restapi.Uri; -import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; -import com.yahoo.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.util.Optional; @@ -71,7 +71,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { private HttpResponse handleOPTIONS() { // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother // spelling out the methods supported at each path, which we should - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,OPTIONS"); return response; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index 5454d71185a..bc360fe3c6f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Set; import java.util.StringJoiner; import java.util.logging.Level; +import java.util.stream.Collectors; /** * This implements the /os/v1 API which provides operators with information about, and scheduling of OS upgrades for @@ -123,7 +125,7 @@ public class OsApiHandler extends AuditLoggingRequestHandler { ZoneList zones = controller.zoneRegistry().zones().controllerUpgraded(); if (path.get("region") != null) zones = zones.in(RegionName.from(path.get("region"))); if (path.get("environment") != null) zones = zones.in(Environment.from(path.get("environment"))); - return zones.ids(); + return zones.zones().stream().map(ZoneApi::getId).collect(Collectors.toList()); } private Slime setOsVersion(HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 5ef997b6d55..7a76f13392d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -13,20 +13,19 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; -import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -99,7 +98,7 @@ public class UserApiHandler extends LoggingRequestHandler { } private HttpResponse handleOPTIONS() { - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); return response; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index fcf01f461a1..6cfaed93fa9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -28,6 +30,8 @@ import java.util.stream.Collectors; @SuppressWarnings("unused") public class ZoneApiHandler extends LoggingRequestHandler { + private static final String OPTIONAL_PREFIX = "/api"; + private final ZoneRegistry zoneRegistry; public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ZoneRegistry zoneRegistry) { @@ -54,7 +58,7 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); + Path path = new Path(request.getUri(), OPTIONAL_PREFIX); if (path.matches("/zone/v1")) { return root(request); } @@ -68,8 +72,8 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse root(HttpRequest request) { - List<Environment> environments = zoneRegistry.zones().all().ids().stream() - .map(ZoneId::environment) + List<Environment> environments = zoneRegistry.zones().all().zones().stream() + .map(ZoneApi::getEnvironment) .distinct() .sorted(Comparator.comparing(Environment::value)) .collect(Collectors.toList()); @@ -88,17 +92,16 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse environment(HttpRequest request, Environment environment) { - List<ZoneId> zones = zoneRegistry.zones().all().in(environment).ids(); Slime slime = new Slime(); Cursor root = slime.setArray(); - zones.forEach(zone -> { + zoneRegistry.zones().all().in(environment).zones().forEach(zone -> { Cursor object = root.addObject(); - object.setString("name", zone.region().value()); + object.setString("name", zone.getRegionName().value()); object.setString("url", request.getUri() .resolve("/zone/v2/environment/") .resolve(environment.value() + "/") .resolve("region/") - .resolve(zone.region().value()) + .resolve(zone.getRegionName().value()) .toString()); }); return new SlimeJsonResponse(slime); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 9d95383fbfb..f0259fc4d51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -94,16 +94,16 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { Cursor root = slime.setObject(); Cursor uris = root.setArray("uris"); ZoneList zoneList = zoneRegistry.zones().reachable(); - zoneList.ids().forEach(zoneId -> uris.addString(request.getUri() + zoneList.zones().forEach(zone -> uris.addString(request.getUri() .resolve("/zone/v2/") - .resolve(zoneId.environment().value() + "/") - .resolve(zoneId.region().value()) + .resolve(zone.getEnvironment().value() + "/") + .resolve(zone.getRegionName().value()) .toString())); Cursor zones = root.setArray("zones"); - zoneList.ids().forEach(zoneId -> { + zoneList.zones().forEach(zone -> { Cursor object = zones.addObject(); - object.setString("environment", zoneId.environment().value()); - object.setString("region", zoneId.region().value()); + object.setString("environment", zone.getEnvironment().value()); + object.setString("region", zone.getRegionName().value()); }); return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index b3953c47c01..f2bc50ec445 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -1,18 +1,30 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.Endpoint; +import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -47,7 +59,12 @@ public class RotationRepository { /** Get rotation for given application */ public Optional<Rotation> getRotation(Application application) { - return application.rotation().map(allRotations::get); + return application.rotations().stream().map(allRotations::get).findFirst(); + } + + /** Get rotation for the given rotationId */ + public Optional<Rotation> getRotation(RotationId rotationId) { + return Optional.of(allRotations.get(rotationId)); } /** @@ -60,8 +77,8 @@ public class RotationRepository { * @param lock Lock which must be acquired by the caller */ public Rotation getOrAssignRotation(Application application, RotationLock lock) { - if (application.rotation().isPresent()) { - return allRotations.get(application.rotation().get()); + if (! application.rotations().isEmpty()) { + return allRotations.get(application.rotations().get(0)); } if (application.deploymentSpec().globalServiceId().isEmpty()) { throw new IllegalArgumentException("global-service-id is not set in deployment spec"); @@ -76,13 +93,123 @@ public class RotationRepository { } /** + * Returns rotation assignments for all endpoints in application. + * + * If rotations are already assigned, these will be returned. + * If rotations are not assigned, a new assignment will be created taking new rotations from the repository. + * This method supports both global-service-id as well as the new endpoints tag. + * + * @param application The application requesting rotations. + * @param lock Lock which by acquired by the caller + * @return List of rotation assignments - either new or existing. + */ + public List<AssignedRotation> getOrAssignRotations(Application application, RotationLock lock) { + if (application.deploymentSpec().globalServiceId().isPresent() && ! application.deploymentSpec().endpoints().isEmpty()) { + throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); + } + + // Support the older case of setting global-service-id + if (application.deploymentSpec().globalServiceId().isPresent()) { + final var regions = application.deploymentSpec().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + + final var rotation = getOrAssignRotation(application, lock); + + return List.of( + new AssignedRotation( + new ClusterSpec.Id(application.deploymentSpec().globalServiceId().get()), + EndpointId.default_(), + rotation.id(), + regions + ) + ); + } + + final Map<EndpointId, AssignedRotation> existingAssignments = existingEndpointAssignments(application); + final Map<EndpointId, AssignedRotation> updatedAssignments = assignRotationsToEndpoints(application, existingAssignments, lock); + + existingAssignments.putAll(updatedAssignments); + + return List.copyOf(existingAssignments.values()); + } + + private Map<EndpointId, AssignedRotation> assignRotationsToEndpoints(Application application, Map<EndpointId, AssignedRotation> existingAssignments, RotationLock lock) { + final var availableRotations = new ArrayList<>(availableRotations(lock).values()); + + final var neededRotations = application.deploymentSpec().endpoints().stream() + .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId())))) + .collect(Collectors.toSet()); + + if (neededRotations.size() > availableRotations.size()) { + throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation: need " + neededRotations.size() + ", have " + availableRotations.size()); + } + + return neededRotations.stream() + .map(endpoint -> { + return new AssignedRotation( + new ClusterSpec.Id(endpoint.containerId()), + EndpointId.of(endpoint.endpointId()), + availableRotations.remove(0).id(), + endpoint.regions() + ); + }) + .collect( + Collectors.toMap( + AssignedRotation::endpointId, + Function.identity(), + (a, b) -> { throw new IllegalStateException("Duplicate entries:" + a + ", " + b); }, + LinkedHashMap::new + ) + ); + } + + private Map<EndpointId, AssignedRotation> existingEndpointAssignments(Application application) { + // + // Get the regions that has been configured for an endpoint. Empty set if the endpoint + // is no longer mentioned in the configuration file. + // + final Function<EndpointId, Set<RegionName>> configuredRegionsForEndpoint = endpointId -> { + return application.deploymentSpec().endpoints().stream() + .filter(endpoint -> endpointId.id().equals(endpoint.endpointId())) + .map(Endpoint::regions) + .findFirst() + .orElse(Set.of()); + }; + + // + // Build a new AssignedRotation instance where we update set of regions from the configuration instead + // of using the one already mentioned in the assignment. This allows us to overwrite the set of regions + // when + final Function<AssignedRotation, AssignedRotation> assignedRotationWithConfiguredRegions = assignedRotation -> { + return new AssignedRotation( + assignedRotation.clusterId(), + assignedRotation.endpointId(), + assignedRotation.rotationId(), + configuredRegionsForEndpoint.apply(assignedRotation.endpointId()) + ); + }; + + return application.assignedRotations().stream() + .collect( + Collectors.toMap( + AssignedRotation::endpointId, + assignedRotationWithConfiguredRegions, + (a, b) -> { throw new IllegalStateException("Duplicate entries: " + a + ", " + b); }, + LinkedHashMap::new + ) + ); + } + + /** * Returns all unassigned rotations * @param lock Lock which must be acquired by the caller */ public Map<RotationId, Rotation> availableRotations(@SuppressWarnings("unused") RotationLock lock) { List<RotationId> assignedRotations = applications.asList().stream() - .filter(application -> application.rotation().isPresent()) - .map(application -> application.rotation().get()) + .filter(application -> ! application.rotations().isEmpty()) + .flatMap(application -> application.rotations().stream()) .collect(Collectors.toList()); Map<RotationId, Rotation> unassignedRotations = new LinkedHashMap<>(this.allRotations); assignedRotations.forEach(unassignedRotations::remove); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index dcc61b13bab..1ac82317695 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -2,15 +2,17 @@ package com.yahoo.vespa.hosted.controller.tls; import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.hosted.controller.tls.config.TlsConfig; -import org.eclipse.jetty.util.ssl.SslContextFactory; import java.nio.file.Files; import java.nio.file.Paths; @@ -28,11 +30,11 @@ import java.util.concurrent.ConcurrentHashMap; * @author bjorncs */ @SuppressWarnings("unused") // Injected -public class ControllerSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { +public class ControllerSslContextFactoryProvider extends TlsContextBasedProvider { private final KeyStore truststore; private final KeyStore keystore; - private final Map<Integer, SslContextFactory> sslContextFactories = new ConcurrentHashMap<>(); + private final Map<Integer, TlsContext> tlsContextMap = new ConcurrentHashMap<>(); @Inject public ControllerSslContextFactoryProvider(SecretStore secretStore, TlsConfig config) { @@ -50,21 +52,17 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple } @Override - public SslContextFactory getInstance(String containerId, int port) { - return sslContextFactories.computeIfAbsent(port, this::createSslContextFactory); + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContextMap.computeIfAbsent(port, this::createTlsContext); } - /** Create a SslContextFactory backed by an in-memory key and trust store */ - private SslContextFactory createSslContextFactory(int port) { - SslContextFactory factory = new SslContextFactory(); - if (port != 443) { - factory.setWantClientAuth(true); - } - factory.setTrustStore(truststore); - factory.setKeyStore(keystore); - factory.setKeyStorePassword(""); - factory.setEndpointIdentificationAlgorithm(null); // disable https hostname verification of clients (must be disabled when using Athenz x509 certificates) - return factory; + private TlsContext createTlsContext(int port) { + return new DefaultTlsContext( + new SslContextBuilder() + .withKeyStore(keystore, new char[0]) + .withTrustStore(truststore) + .build(), + port != 443 ? PeerAuthentication.WANT : PeerAuthentication.DISABLED); } /** Get private key from secret store **/ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java new file mode 100644 index 00000000000..9f3addd4992 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java @@ -0,0 +1,64 @@ +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; +import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; +import com.yahoo.vespa.hosted.controller.api.integration.maven.Metadata; +import com.yahoo.vespa.hosted.controller.maven.repository.config.MavenRepositoryConfig; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Http client implementation of a {@link MavenRepository}, which uses a configured repository and artifact ID. + * + * @author jonmv + */ +public class MavenRepositoryClient implements MavenRepository { + + private final HttpClient client; + private final URI apiUrl; + private final ArtifactId id; + + public MavenRepositoryClient(MavenRepositoryConfig config) { + this.client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + this.apiUrl = URI.create(config.apiUrl() + "/").normalize(); + this.id = new ArtifactId(config.groupId(), config.artifactId()); + } + + @Override + public Metadata metadata() { + try { + HttpRequest request = HttpRequest.newBuilder(withArtifactPath(apiUrl, id)).build(); + HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString(UTF_8)); + if (response.statusCode() != 200) + throw new RuntimeException("Status code '" + response.statusCode() + "' and body\n'''\n" + + response.body() + "\n'''\nfor request " + request); + + return Metadata.fromXml(response.body()); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public ArtifactId artifactId() { + return id; + } + + static URI withArtifactPath(URI baseUrl, ArtifactId id) { + List<String> parts = new ArrayList<>(List.of(id.groupId().split("\\."))); + parts.add(id.artifactId()); + parts.add("maven-metadata.xml"); + return baseUrl.resolve(String.join("/", parts)); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index a18c1f47036..f5b9d8263e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -67,27 +68,25 @@ public class OsVersionStatus { controller.osVersions().forEach(osVersion -> versions.put(osVersion, new ArrayList<>())); for (SystemApplication application : SystemApplication.all()) { - if (application.nodeTypesWithUpgradableOs().isEmpty()) { - continue; // Avoid querying applications that do not contain nodes with upgradable OS + if (!application.isEligibleForOsUpgrades()) { + continue; // Avoid querying applications that are not eligible for OS upgrades } - for (ZoneId zone : zonesToUpgrade(controller)) { - controller.configServer().nodeRepository().list(zone, application.id()).stream() + for (ZoneApi zone : zonesToUpgrade(controller)) { + controller.configServer().nodeRepository().list(zone.getId(), application.id()).stream() .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) - .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.environment(), zone.region())) - .forEach(node -> versions.compute(new OsVersion(node.version(), zone.cloud()), (ignored, nodes) -> { - if (nodes == null) { - nodes = new ArrayList<>(); - } - nodes.add(node); - return nodes; - })); + .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.getEnvironment(), zone.getRegionName())) + .forEach(node -> { + var version = new OsVersion(node.version(), zone.getCloudName()); + versions.putIfAbsent(version, new ArrayList<>()); + versions.get(version).add(node); + }); } } return new OsVersionStatus(versions); } - private static List<ZoneId> zonesToUpgrade(Controller controller) { + private static List<ZoneApi> zonesToUpgrade(Controller controller) { return controller.zoneRegistry().osUpgradePolicies().stream() .flatMap(upgradePolicy -> upgradePolicy.asList().stream()) .flatMap(Collection::stream) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index e2d4c90f443..ab5fd2714e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -6,12 +6,13 @@ import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobList; @@ -129,14 +130,17 @@ public class VersionStatus { Collection<DeploymentStatistics> deploymentStatistics = computeDeploymentStatistics(infrastructureVersions, controller.applications().asList()); List<VespaVersion> versions = new ArrayList<>(); + List<Version> releasedVersions = controller.mavenRepository().metadata().versions(); for (DeploymentStatistics statistics : deploymentStatistics) { if (statistics.version().isEmpty()) continue; try { + boolean isReleased = Collections.binarySearch(releasedVersions, statistics.version()) >= 0; VespaVersion vespaVersion = createVersion(statistics, statistics.version().equals(controllerVersion), statistics.version().equals(systemVersion), + isReleased, systemApplicationVersions.getList(statistics.version()), controller); versions.add(vespaVersion); @@ -145,25 +149,28 @@ public class VersionStatus { statistics.version().toFullString(), e); } } + Collections.sort(versions); return new VersionStatus(versions); } private static ListMap<Version, HostName> findSystemApplicationVersions(Controller controller) { - List<ZoneId> zones = controller.zoneRegistry().zones() - .controllerUpgraded() - .ids(); ListMap<Version, HostName> versions = new ListMap<>(); - for (ZoneId zone : zones) { + for (ZoneApi zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { for (SystemApplication application : SystemApplication.all()) { - boolean configConverged = application.configConvergedIn(zone, controller); + List<Node> eligibleForUpgradeApplicationNodes = controller.configServer().nodeRepository() + .list(zone.getId(), application.id()).stream() + .filter(SystemUpgrader::eligibleForUpgrade) + .collect(Collectors.toList()); + if (eligibleForUpgradeApplicationNodes.isEmpty()) + continue; + + boolean configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty()); if (!configConverged) { log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone + " has not converged"); } - for (Node node : controller.configServer().nodeRepository().list(zone, application.id()).stream() - .filter(SystemUpgrader::eligibleForUpgrade) - .collect(Collectors.toList())) { + for (Node node : eligibleForUpgradeApplicationNodes) { // Only use current node version if config has converged Version nodeVersion = configConverged ? node.currentVersion() : controller.systemVersion(); versions.put(nodeVersion, node.hostname()); @@ -233,10 +240,11 @@ public class VersionStatus { } return versionMap.values(); } - + private static VespaVersion createVersion(DeploymentStatistics statistics, boolean isControllerVersion, - boolean isSystemVersion, + boolean isSystemVersion, + boolean isReleased, Collection<HostName> configServerHostnames, Controller controller) { GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString()); @@ -255,6 +263,7 @@ public class VersionStatus { gitSha.sha, committedAt, isControllerVersion, isSystemVersion, + isReleased, configServerHostnames, confidence ); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index ffbf24be12a..117ce80adaa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -27,12 +27,13 @@ public class VespaVersion implements Comparable<VespaVersion> { private final Instant committedAt; private final boolean isControllerVersion; private final boolean isSystemVersion; + private final boolean isReleased; private final DeploymentStatistics statistics; private final ImmutableSet<HostName> systemApplicationHostnames; private final Confidence confidence; public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant committedAt, - boolean isControllerVersion, boolean isSystemVersion, + boolean isControllerVersion, boolean isSystemVersion, boolean isReleased, Collection<HostName> systemApplicationHostnames, Confidence confidence) { this.statistics = statistics; @@ -40,6 +41,7 @@ public class VespaVersion implements Comparable<VespaVersion> { this.committedAt = committedAt; this.isControllerVersion = isControllerVersion; this.isSystemVersion = isSystemVersion; + this.isReleased = isReleased; this.systemApplicationHostnames = ImmutableSet.copyOf(systemApplicationHostnames); this.confidence = confidence; } @@ -102,6 +104,9 @@ public class VespaVersion implements Comparable<VespaVersion> { */ public boolean isSystemVersion() { return isSystemVersion; } + /** Returns whether the artifacts of this release are available in the configured maven repository. */ + public boolean isReleased() { return isReleased; } + /** Returns the hosts allocated to system applications (across all zones) which are currently of this version */ public Set<HostName> systemApplicationHostnames() { return systemApplicationHostnames; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java new file mode 100644 index 00000000000..c6f2c1e427d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.osgi.annotation.ExportPackage; |