diff options
Diffstat (limited to 'controller-server')
24 files changed, 348 insertions, 227 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index a382d357e49..e02c1b9f5dd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; @@ -17,7 +18,10 @@ import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; @@ -108,6 +112,7 @@ public class ApplicationController { private final DeploymentTrigger deploymentTrigger; private final ApplicationPackageValidator applicationPackageValidator; private final EndpointCertificateManager endpointCertificateManager; + private final StringFlag dockerImageRepoFlag; ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock, SecretStore secretStore, FlagSource flagSource) { @@ -119,6 +124,7 @@ public class ApplicationController { this.clock = clock; this.artifactRepository = controller.serviceRegistry().artifactRepository(); this.applicationStore = controller.serviceRegistry().applicationStore(); + this.dockerImageRepoFlag = Flags.DOCKER_IMAGE_REPO.bindTo(flagSource); deploymentTrigger = new DeploymentTrigger(controller, clock); applicationPackageValidator = new ApplicationPackageValidator(controller); @@ -129,28 +135,18 @@ public class ApplicationController { Once.after(Duration.ofMinutes(1), () -> { Instant start = clock.instant(); int count = 0; - for (TenantAndApplicationId id: curator.readApplicationIds()) { + for (TenantAndApplicationId id : curator.readApplicationIds()) { lockApplicationIfPresent(id, application -> { - if (id.tenant().value().startsWith("by-")) { // TODO jonmv: Remove after run once. - for (Instance instance : application.get().instances().values()) - for (ZoneId zone : instance.deployments().keySet()) - deactivate(instance.id(), zone); - curator.removeApplication(id); - } - else { - for (InstanceName instance : application.get().deploymentSpec().instanceNames()) - if (!application.get().instances().containsKey(instance)) - application = withNewInstance(application, id.instance(instance)); - store(application); - } + for (InstanceName instance : application.get().deploymentSpec().instanceNames()) + if (!application.get().instances().containsKey(instance)) + application = withNewInstance(application, id.instance(instance)); + store(application); }); count++; } log.log(Level.INFO, String.format("Wrote %d applications in %s", count, Duration.between(start, clock.instant()))); }); - - // TODO jonmv: Do the above for applications as well when they split writes. } /** Returns the application with the given id, or null if it is not present */ @@ -185,7 +181,18 @@ public class ApplicationController { /** Returns a snapshot of all applications */ public List<Application> asList() { - return curator.readApplications(); + return curator.readApplications(false); + } + + /** + * Returns a snapshot of all readable applications. Unlike {@link ApplicationController#asList()} this ignores + * applications that cannot currently be read (e.g. due to serialization issues) and may return an incomplete + * snapshot. + * + * This should only be used in cases where acting on a subset of applications is better than none. + */ + public List<Application> readable() { + return curator.readApplications(true); } /** Returns the ID of all known applications. */ @@ -485,9 +492,21 @@ public class ApplicationController { ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata) { try { + Optional<DockerImage> dockerImageRepo = Optional.ofNullable( + dockerImageRepoFlag + .with(FetchVector.Dimension.ZONE_ID, zone.value()) + .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) + .value()) + .filter(s -> !s.isBlank()) + .map(DockerImage::fromString); + + Optional<AthenzDomain> domain = controller.tenants().get(application.tenant()) + .filter(tenant-> tenant instanceof AthenzTenant) + .map(tenant -> ((AthenzTenant)tenant).domain()); + ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, - endpoints, endpointCertificateMetadata)); + endpoints, endpointCertificateMetadata, dockerImageRepo, domain)); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 5189721df5d..017b8cfaf45 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -13,8 +13,6 @@ import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSnapshot; -import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSource; import com.yahoo.vespa.hosted.controller.api.integration.ServiceRegistry; import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; @@ -55,7 +53,7 @@ import java.util.stream.Collectors; * * @author bratseth */ -public class Controller extends AbstractComponent implements ApplicationIdSource { +public class Controller extends AbstractComponent { private static final Logger log = Logger.getLogger(Controller.class.getName()); @@ -272,15 +270,4 @@ public class Controller extends AbstractComponent implements ApplicationIdSource return vespaVersion.map(v -> v.versionNumber().toFullString()).orElse("unknown"); } - @Override - public ApplicationIdSnapshot applicationIdSnapshot() { - ApplicationIdSnapshot.Builder snapshotBuilder = new ApplicationIdSnapshot.Builder(); - tenants().asList().forEach(tenant -> snapshotBuilder.add(tenant.name())); - applications().asList().forEach(application -> { - snapshotBuilder.add(application.id().tenant(), application.id().application()); - application.instances().values().stream().map(Instance::id).forEach(snapshotBuilder::add); - }); - - return snapshotBuilder.build(); - } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 9efb745ddf6..df62c501cc6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; @@ -64,6 +65,7 @@ public class RoutingController { private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; private final BooleanFlag allowDirectRouting; + private final BooleanFlag disableRoutingGenerator; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -71,6 +73,7 @@ public class RoutingController { this.rotationRepository = new RotationRepository(rotationsConfig, controller.applications(), controller.curator()); this.allowDirectRouting = Flags.ALLOW_DIRECT_ROUTING.bindTo(controller.flagSource()); + this.disableRoutingGenerator = Flags.DISABLE_ROUTING_GENERATOR.bindTo(controller.flagSource()); } public RoutingPolicies policies() { @@ -84,12 +87,27 @@ public class RoutingController { /** Returns zone-scoped endpoints for given deployment */ public EndpointList endpointsOf(DeploymentId deployment) { var endpoints = new LinkedHashSet<Endpoint>(); + // TODO(mpolden): Remove this once all applications have deployed once and config server passes correct cluster + // id for combined cluster type + var disableRoutingGenerator = this.disableRoutingGenerator.with(FetchVector.Dimension.APPLICATION_ID, + deployment.applicationId().serializedForm()) + .value(); + if (!disableRoutingGenerator) { + controller.serviceRegistry().routingGenerator().clusterEndpoints(deployment) + .forEach((cluster, url) -> endpoints.add(Endpoint.of(deployment.applicationId()) + .target(cluster, deployment.zoneId()) + .routingMethod(RoutingMethod.shared) + .on(Port.fromRoutingMethod(RoutingMethod.shared)) + .in(controller.system()))); + } + boolean hasSharedEndpoint = !endpoints.isEmpty(); // Avoid reading application more than once per call to this var application = Suppliers.memoize(() -> controller.applications().requireApplication(TenantAndApplicationId.from(deployment.applicationId()))); for (var policy : routingPolicies.get(deployment).values()) { if (!policy.status().isActive()) continue; for (var routingMethod : controller.zoneRegistry().routingMethods(policy.id().zone())) { if (routingMethod.isDirect() && !canRouteDirectlyTo(deployment, application.get())) continue; + if (hasSharedEndpoint && routingMethod == RoutingMethod.shared) continue; endpoints.add(policy.endpointIn(controller.system(), routingMethod)); } } @@ -103,8 +121,6 @@ public class RoutingController { } /** Returns global-scoped endpoints for given instance */ - // TODO(mpolden): Add a endpointsOf(Instance, DeploymentId) variant of this that only returns global endpoint of - // which deployment is a member public EndpointList endpointsOf(Application application, InstanceName instanceName) { var endpoints = new LinkedHashSet<Endpoint>(); // Add global endpoints provided by rotations @@ -113,10 +129,9 @@ public class RoutingController { var deployments = rotation.regions().stream() .map(region -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, region))) .collect(Collectors.toList()); - EndpointList.global(RoutingId.of(instance.id(), rotation.endpointId()), - controller.system(), routingMethodsOfAll(deployments, application)) - .requiresRotation() - .forEach(endpoints::add); + computeGlobalEndpoints(RoutingId.of(instance.id(), rotation.endpointId()), + application, deployments).requiresRotation() + .forEach(endpoints::add); } // Add global endpoints provided by routing policies var deploymentsByRoutingId = new LinkedHashMap<RoutingId, List<DeploymentId>>(); @@ -129,9 +144,7 @@ public class RoutingController { } } deploymentsByRoutingId.forEach((routingId, deployments) -> { - EndpointList.global(routingId, controller.system(), routingMethodsOfAll(deployments, application)) - .not().requiresRotation() - .forEach(endpoints::add); + computeGlobalEndpoints(routingId, application, deployments).not().requiresRotation().forEach(endpoints::add); }); return EndpointList.copyOf(endpoints); } @@ -293,4 +306,38 @@ public class RoutingController { .value(); } + /** Compute global endpoints for given routing ID, application and deployments */ + private EndpointList computeGlobalEndpoints(RoutingId routingId, Application application, List<DeploymentId> deployments) { + var endpoints = new ArrayList<Endpoint>(); + var directMethods = 0; + var targets = deployments.stream().map(DeploymentId::zoneId).collect(Collectors.toList()); + for (var method : routingMethodsOfAll(deployments, application)) { + if (method.isDirect() && ++directMethods > 1) { + throw new IllegalArgumentException("Invalid routing methods for " + routingId + ": Exceeded maximum " + + "direct methods"); + } + endpoints.add(Endpoint.of(routingId.application()) + .named(routingId.endpointId(), targets) + .on(Port.fromRoutingMethod(method)) + .routingMethod(method) + .in(controller.system())); + // TODO(mpolden): Remove this once all applications have migrated away from legacy endpoints + if (method == RoutingMethod.shared) { + endpoints.add(Endpoint.of(routingId.application()) + .named(routingId.endpointId(), targets) + .on(Port.plain(4080)) + .legacy() + .routingMethod(method) + .in(controller.system())); + endpoints.add(Endpoint.of(routingId.application()) + .named(routingId.endpointId(), targets) + .on(Port.tls(4443)) + .legacy() + .routingMethod(method) + .in(controller.system())); + } + } + return EndpointList.copyOf(endpoints); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 31394ef8d33..ed5add8b98a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import java.net.URI; +import java.util.List; import java.util.Objects; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -29,26 +30,31 @@ public class Endpoint { private final String name; private final URI url; + private final List<ZoneId> zones; private final Scope scope; private final boolean legacy; private final RoutingMethod routingMethod; private final boolean tls; - private final boolean wildcard; - private Endpoint(String name, ApplicationId application, ZoneId zone, SystemName system, Port port, boolean legacy, - RoutingMethod routingMethod, boolean wildcard) { + private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system, + Port port, boolean legacy, RoutingMethod routingMethod) { Objects.requireNonNull(name, "name must be non-null"); Objects.requireNonNull(application, "application must be non-null"); + Objects.requireNonNull(zones, "zones must be non-null"); + Objects.requireNonNull(scope, "scope must be non-null"); Objects.requireNonNull(system, "system must be non-null"); Objects.requireNonNull(port, "port must be non-null"); Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); + if (scope == Scope.zone && zones.size() != 1) { + throw new IllegalArgumentException("A single zone must be given for zone-scoped endpoints"); + } this.name = name; - this.url = createUrl(name, application, zone, system, port, legacy, routingMethod); - this.scope = zone == null ? Scope.global : Scope.zone; + this.url = createUrl(name, application, zones, scope, system, port, legacy, routingMethod); + this.zones = List.copyOf(zones); + this.scope = scope; this.legacy = legacy; this.routingMethod = routingMethod; this.tls = port.tls; - this.wildcard = wildcard; } /** @@ -73,6 +79,11 @@ public class Endpoint { return url.getAuthority().replaceAll(":.*", ""); } + /** Returns the zone(s) to which this routes traffic */ + public List<ZoneId> zones() { + return zones; + } + /** Returns the scope of this */ public Scope scope() { return scope; @@ -98,11 +109,6 @@ public class Endpoint { return routingMethod.isShared() && scope == Scope.global; } - /** Returns whether this is a wildcard endpoint (used only in certificates) */ - public boolean wildcard() { - return wildcard; - } - /** Returns the upstream ID of given deployment. This *must* match what the routing layer generates */ public String upstreamIdOf(DeploymentId deployment) { if (scope != Scope.global) throw new IllegalArgumentException("Scope " + scope + " does not have upstream name"); @@ -133,8 +139,8 @@ public class Endpoint { return dnsSuffix(system, false); } - private static URI createUrl(String name, ApplicationId application, ZoneId zone, SystemName system, - Port port, boolean legacy, RoutingMethod routingMethod) { + private static URI createUrl(String name, ApplicationId application, List<ZoneId> zones, Scope scope, + SystemName system, Port port, boolean legacy, RoutingMethod routingMethod) { String scheme = port.tls ? "https" : "http"; String separator = separator(system, routingMethod, port.tls); String portPart = port.isDefault() ? "" : ":" + port.port; @@ -146,7 +152,7 @@ public class Endpoint { separator + sanitize(application.tenant().value()) + "." + - scopePart(zone, legacy) + + scopePart(scope, zones, legacy) + dnsSuffix(system, legacy) + portPart + "/"); @@ -168,8 +174,9 @@ public class Endpoint { return name + separator; } - private static String scopePart(ZoneId zone, boolean legacy) { - if (zone == null) return "global"; + private static String scopePart(Scope scope, List<ZoneId> zones, boolean legacy) { + if (scope == Scope.global) return "global"; + var zone = zones.get(0); if (!legacy && zone.environment().isProduction()) return zone.region().value(); // Skip prod environment for non-legacy endpoints return zone.region().value() + "." + zone.environment().value(); } @@ -283,7 +290,8 @@ public class Endpoint { private final ApplicationId application; - private ZoneId zone; + private Scope scope; + private List<ZoneId> zones; private ClusterSpec.Id cluster; private EndpointId endpointId; private Port port; @@ -301,35 +309,44 @@ public class Endpoint { throw new IllegalArgumentException("Cannot set multiple target types"); } this.cluster = cluster; - this.zone = zone; + this.scope = Scope.zone; + this.zones = List.of(zone); return this; } /** Sets the endpoint target ID for this (as defined in deployments.xml) */ public EndpointBuilder named(EndpointId endpointId) { + return named(endpointId, List.of()); + } + + /** Sets the endpoint ID for this (as defined in deployments.xml) */ + public EndpointBuilder named(EndpointId endpointId, List<ZoneId> targets) { if (cluster != null || wildcard) { throw new IllegalArgumentException("Cannot set multiple target types"); } this.endpointId = endpointId; + this.zones = targets; + this.scope = Scope.global; return this; } /** Sets the global wildcard target for this */ public EndpointBuilder wildcard() { - if (endpointId != null || cluster != null) { - throw new IllegalArgumentException("Cannot set multiple target types"); - } - this.wildcard = true; - return this; + return wildcard(Scope.global, List.of()); } /** Sets the zone wildcard target for this */ public EndpointBuilder wildcard(ZoneId zone) { - if(endpointId != null || cluster != null) { + return wildcard(Scope.zone, List.of(zone)); + } + + private EndpointBuilder wildcard(Scope scope, List<ZoneId> zones) { + if (endpointId != null || cluster != null) { throw new IllegalArgumentException("Cannot set multiple target types"); } - this.zone = zone; this.wildcard = true; + this.scope = scope; + this.zones = zones; return this; } @@ -369,7 +386,7 @@ public class Endpoint { if (routingMethod.isDirect() && !port.isDefault()) { throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port"); } - return new Endpoint(name, application, zone, system, port, legacy, routingMethod, wildcard); + return new Endpoint(name, application, zones, scope, system, port, legacy, routingMethod); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index a4c5570ee3f..4da34dad737 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -2,12 +2,8 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.collections.AbstractFilteringList; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; -import com.yahoo.vespa.hosted.controller.routing.RoutingId; +import com.yahoo.config.provision.zone.ZoneId; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -27,13 +23,9 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> } } - private EndpointList(Collection<? extends Endpoint> endpoints) { - this(endpoints, false); - } - /** Returns the primary (non-legacy) endpoint, if any */ public Optional<Endpoint> primary() { - return not().matching(Endpoint::legacy).asList().stream().findFirst(); + return not().legacy().asList().stream().findFirst(); } /** Returns the subset of endpoints named according to given ID */ @@ -41,6 +33,16 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return matching(endpoint -> endpoint.name().equals(id.id())); } + /** Returns the subset of endpoints which target all of the given zones */ + public EndpointList targets(List<ZoneId> zones) { + return matching(endpoint -> endpoint.zones().containsAll(zones)); + } + + /** Returns the subset of endpoints which target the given zones */ + public EndpointList targets(ZoneId zone) { + return targets(List.of(zone)); + } + /** Returns the subset of endpoints that are considered legacy */ public EndpointList legacy() { return matching(Endpoint::legacy); @@ -56,45 +58,8 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return matching(endpoint -> endpoint.scope() == scope); } - /** Returns all global endpoints for given routing ID and system provided by given routing methods */ - public static EndpointList global(RoutingId routingId, SystemName system, List<RoutingMethod> routingMethods) { - var endpoints = new ArrayList<Endpoint>(); - var directMethods = 0; - for (var method : routingMethods) { - if (method.isDirect() && ++directMethods > 1) { - throw new IllegalArgumentException("Invalid routing methods for " + routingId + ": Exceeded maximum " + - "direct methods, got " + routingMethods); - } - endpoints.add(Endpoint.of(routingId.application()) - .named(routingId.endpointId()) - .on(Port.fromRoutingMethod(method)) - .routingMethod(method) - .in(system)); - // TODO(mpolden): Remove this once all applications have migrated away from legacy endpoints - if (method == RoutingMethod.shared) { - endpoints.add(Endpoint.of(routingId.application()) - .named(routingId.endpointId()) - .on(Port.plain(4080)) - .legacy() - .routingMethod(method) - .in(system)); - endpoints.add(Endpoint.of(routingId.application()) - .named(routingId.endpointId()) - .on(Port.tls(4443)) - .legacy() - .routingMethod(method) - .in(system)); - } - } - return new EndpointList(endpoints); - } - - public static EndpointList global(RoutingId routingId, SystemName system, RoutingMethod routingMethod) { - return global(routingId, system, List.of(routingMethod)); - } - public static EndpointList copyOf(Collection<Endpoint> endpoints) { - return new EndpointList(endpoints); + return new EndpointList(endpoints, false); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 615be3bcde1..9b71439e01d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -263,7 +263,7 @@ public class DeploymentTrigger { /** Returns the set of all jobs which have changes to propagate from the upstream steps. */ private List<Job> computeReadyJobs() { - return jobs.deploymentStatuses(ApplicationList.from(applications().asList()) + return jobs.deploymentStatuses(ApplicationList.from(applications().readable()) .withProjectId() // Need to keep this, as we have applications with deployment spec that shouldn't be orchestrated. .withDeploymentSpec()) .withChanges() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index e696cb2d033..7c5c8d55e2a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -194,16 +194,9 @@ public class JobController { locked(id, run -> run.with(testerCertificate)); } - /** Returns a list of all applications which have registered. */ - public List<TenantAndApplicationId> applications() { - return copyOf(controller.applications().asList().stream() - .map(Application::id) - .iterator()); - } - /** Returns a list of all instances of applications which have registered. */ public List<ApplicationId> instances() { - return copyOf(controller.applications().asList().stream() + return copyOf(controller.applications().readable().stream() .flatMap(application -> application.instances().values().stream()) .map(Instance::id) .iterator()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 312d885cf53..9c426d23d82 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.organization.ApplicationSummary; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; @@ -46,7 +46,7 @@ public class ApplicationOwnershipConfirmer extends Maintainer { /** File an ownership issue with the owners of all applications we know about. */ private void confirmApplicationOwnerships() { - ApplicationList.from(controller().applications().asList()) + applications() .withProjectId() .withProductionDeployment() .asList() @@ -86,7 +86,7 @@ public class ApplicationOwnershipConfirmer extends Maintainer { /** Escalate ownership issues which have not been closed before a defined amount of time has passed. */ private void ensureConfirmationResponses() { - for (Application application : controller().applications().asList()) + for (Application application : applications()) application.ownershipIssueId().ifPresent(issueId -> { try { Tenant tenant = tenantOf(application.id()); @@ -99,7 +99,7 @@ public class ApplicationOwnershipConfirmer extends Maintainer { } private void updateConfirmedApplicationOwners() { - ApplicationList.from(controller().applications().asList()) + applications() .withProjectId() .withProductionDeployment() .asList() @@ -114,6 +114,10 @@ public class ApplicationOwnershipConfirmer extends Maintainer { }); } + private ApplicationList applications() { + return ApplicationList.from(controller().applications().readable()); + } + private User determineAssignee(Application application) { return application.owner().orElse(null); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 7756e6b23a7..3160e7aef1d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -24,7 +24,7 @@ public class DeploymentExpirer extends Maintainer { @Override protected void maintain() { - for (Application application : controller().applications().asList()) + for (Application application : controller().applications().readable()) for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) { if ( ! isExpired(deployment)) continue; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 3963040f6b5..94c3f3af98e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -51,7 +51,7 @@ public class DeploymentIssueReporter extends Maintainer { /** Returns the applications to maintain issue status for. */ private List<Application> applications() { - return ApplicationList.from(controller().applications().asList()) + return ApplicationList.from(controller().applications().readable()) .withProjectId() .asList(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 808961257ad..1ce1e7f4c0c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -46,7 +46,7 @@ public class DeploymentMetricsMaintainer extends Maintainer { // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used ForkJoinPool pool = new ForkJoinPool(applicationsToUpdateInParallel); pool.submit(() -> - applications.asList().parallelStream().forEach(application -> { + applications.readable().parallelStream().forEach(application -> { for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) { attempts.incrementAndGet(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index a4ace4cf518..4c1dd56ee64 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -71,8 +71,8 @@ public class MetricsReporter extends Maintainer { } private void reportDeploymentMetrics() { - ApplicationList applications = ApplicationList.from(controller().applications().asList()) - .withProductionDeployment(); + ApplicationList applications = ApplicationList.from(controller().applications().readable()) + .withProductionDeployment(); DeploymentStatusList deployments = controller().jobController().deploymentStatuses(applications); metric.set(DEPLOYMENT_FAIL_METRIC, deploymentFailRatio(deployments) * 100, metric.createContext(Map.of())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java index 01bafc25816..83deb71e663 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployer.java @@ -20,7 +20,7 @@ public class OutstandingChangeDeployer extends Maintainer { @Override protected void maintain() { - for (Application application : ApplicationList.from(controller().applications().asList()) + for (Application application : ApplicationList.from(controller().applications().readable()) .withProductionDeployment() .withDeploymentSpec() .asList()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java index 1d1d5e93dd8..35131d3db80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java @@ -44,9 +44,9 @@ public class RotationStatusUpdater extends Maintainer { var failures = new AtomicInteger(0); var attempts = new AtomicInteger(0); var lastException = new AtomicReference<Exception>(null); - var instancesWithRotations = ApplicationList.from(applications.asList()).hasRotation().asList().stream() - .flatMap(application -> application.instances().values().stream()) - .filter(instance -> ! instance.rotations().isEmpty()); + var instancesWithRotations = ApplicationList.from(applications.readable()).hasRotation().asList().stream() + .flatMap(application -> application.instances().values().stream()) + .filter(instance -> ! instance.rotations().isEmpty()); // Run parallel stream inside a custom ForkJoinPool so that we can control the number of threads used var pool = new ForkJoinPool(applicationsToUpdateInParallel); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index ec975a6a5d5..1e22c59636a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -103,7 +103,7 @@ public class Upgrader extends Maintainer { /** Returns a list of all production application instances, except those which are pinned, which we should not manipulate here. */ private InstanceList instances() { - return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().asList()))) + return InstanceList.from(controller().jobController().deploymentStatuses(ApplicationList.from(controller().applications().readable()))) .withProductionDeployment() .unpinned(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index eb86b1028e2..4fdd66ee100 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.log.LogLevel; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; @@ -18,13 +19,13 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import com.yahoo.vespa.hosted.controller.tenant.Tenant; @@ -38,6 +39,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -352,26 +354,37 @@ public class CuratorDb { return read(applicationPath(application), applicationSerializer::fromSlime); } - public List<Application> readApplications() { - return readApplications(ignored -> true); + public List<Application> readApplications(boolean canFail) { + return readApplications(ignored -> true, canFail); } public List<Application> readApplications(TenantName name) { - return readApplications(application -> application.tenant().equals(name)); - } - - private List<Application> readApplications(Predicate<TenantAndApplicationId> applicationFilter) { - return readApplicationIds().stream() - .filter(applicationFilter) - .sorted() - .map(this::readApplication) - .flatMap(Optional::stream) - .collect(Collectors.toUnmodifiableList()); + return readApplications(application -> application.tenant().equals(name), false); + } + + private List<Application> readApplications(Predicate<TenantAndApplicationId> applicationFilter, boolean canFail) { + var applicationIds = readApplicationIds(); + var applications = new ArrayList<Application>(applicationIds.size()); + for (var id : applicationIds) { + if (!applicationFilter.test(id)) continue; + try { + readApplication(id).ifPresent(applications::add); + } catch (Exception e) { + if (canFail) { + log.log(LogLevel.ERROR, "Failed to read application '" + id + "', this must be fixed through " + + "manual intervention", e); + } else { + throw e; + } + } + } + return Collections.unmodifiableList(applications); } public List<TenantAndApplicationId> readApplicationIds() { return curator.getChildren(applicationRoot).stream() .map(TenantAndApplicationId::fromSerialized) + .sorted() .collect(toUnmodifiableList()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 8c2b2d043d4..be3f4e50dc7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1070,11 +1070,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } } // Add global endpoints - if (deploymentId.zoneId().environment().isProduction()) { // Global endpoints can only point to production deployments - for (var endpoint : controller.routing().endpointsOf(application, deploymentId.applicationId().instance()).not().legacy()) { - // TODO(mpolden): Pass cluster name. Cluster that a global endpoint points to is not available at this level. - toSlime(endpoint, "", endpointArray.addObject()); - } + var globalEndpoints = controller.routing().endpointsOf(application, deploymentId.applicationId().instance()) + .not().legacy() + .targets(deploymentId.zoneId()); + for (var endpoint : globalEndpoints) { + // TODO(mpolden): Pass cluster name. Cluster that a global endpoint points to is not available at this level. + toSlime(endpoint, "", endpointArray.addObject()); } // TODO(mpolden): Remove this once all clients stop reading it Cursor serviceUrlArray = response.setArray("serviceUrls"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index d27bc581f75..5dde9516ab5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -89,7 +89,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { Cursor platformArray = root.setArray("versions"); var versionStatus = controller.versionStatus(); var systemVersion = versionStatus.systemVersion().map(VespaVersion::versionNumber).orElse(Vtag.currentVersion); - Map<ApplicationId, JobList> jobs = controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()), systemVersion) + Map<ApplicationId, JobList> jobs = controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().readable()), systemVersion) .asList().stream() .flatMap(status -> status.instanceJobs().entrySet().stream()) .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index 53ef786d3f6..71b1fb9b2e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -1,10 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; +import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.Endpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; @@ -65,27 +66,30 @@ public class RotationRepository { * If a rotation is already assigned to the application, that rotation will be returned. * If no rotation is assigned, return an available rotation. The caller is responsible for assigning the rotation. * - * @param deploymentSpec the deployment spec for the application + * @param instanceSpec the instance deployment spec * @param instance the instance requesting a rotation * @param lock lock which must be acquired by the caller */ - private Rotation getOrAssignRotation(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { - if ( ! instance.rotations().isEmpty()) { - return allRotations.get(instance.rotations().get(0).rotationId()); + private AssignedRotation assignRotationTo(String globalServiceId, DeploymentInstanceSpec instanceSpec, + Instance instance, RotationLock lock) { + RotationId rotation; + if (instance.rotations().isEmpty()) { + rotation = findAvailableRotation(instance.id(), lock).id(); + } else { + rotation = instance.rotations().get(0).rotationId(); } - - if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isEmpty()) { - throw new IllegalArgumentException("global-service-id is not set in deployment spec for instance '" + - instance.name() + "'"); - } - long productionZones = deploymentSpec.requireInstance(instance.name()).zones().stream() - .filter(zone -> zone.concerns(Environment.prod)) - .count(); - if (productionZones < 2) { + var productionRegions = instanceSpec.zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + if (productionRegions.size() < 2) { throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined " + "in instance '" + instance.name() + "'"); } - return findAvailableRotation(instance.id(), lock); + return new AssignedRotation(new ClusterSpec.Id(globalServiceId), + EndpointId.defaultId(), + rotation, + productionRegions); } /** @@ -107,40 +111,28 @@ public class RotationRepository { } // Only allow one kind of configuration syntax - if ( deploymentSpec.requireInstance(instance.name()).globalServiceId().isPresent() - && ! deploymentSpec.requireInstance(instance.name()).endpoints().isEmpty()) { + var instanceSpec = deploymentSpec.requireInstance(instance.name()); + if ( instanceSpec.globalServiceId().isPresent() + && ! instanceSpec.endpoints().isEmpty()) { throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); } - // Support the older case of setting global-service-id - if (deploymentSpec.requireInstance(instance.name()).globalServiceId().isPresent()) { - var regions = deploymentSpec.requireInstance(instance.name()).zones().stream() - .filter(zone -> zone.environment().isProduction()) - .flatMap(zone -> zone.region().stream()) - .collect(Collectors.toSet()); - - var rotation = getOrAssignRotation(deploymentSpec, instance, lock); - - return List.of( - new AssignedRotation( - new ClusterSpec.Id(deploymentSpec.requireInstance(instance.name()).globalServiceId().get()), - EndpointId.defaultId(), - rotation.id(), - regions - ) - ); + // Support legacy global-service-id + if (instanceSpec.globalServiceId().isPresent()) { + return List.of(assignRotationTo(instanceSpec.globalServiceId().get(), instanceSpec, instance, lock)); } - return assignRotations(deploymentSpec, instance, lock); + return assignRotationsTo(instanceSpec.endpoints(), instance, lock); } - private List<AssignedRotation> assignRotations(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { + private List<AssignedRotation> assignRotationsTo(List<Endpoint> endpoints, Instance instance, RotationLock lock) { + if (endpoints.isEmpty()) return List.of(); // No endpoints declared, nothing to assign. var availableRotations = new ArrayList<>(availableRotations(lock).values()); var assignedRotationsByEndpointId = instance.rotations().stream() .collect(Collectors.toMap(AssignedRotation::endpointId, Function.identity())); var assignments = new ArrayList<AssignedRotation>(); - for (var endpoint : deploymentSpec.requireInstance(instance.name()).endpoints()) { + for (var endpoint : endpoints) { var endpointId = EndpointId.of(endpoint.endpointId()); var assignedRotation = assignedRotationsByEndpointId.get(endpointId); RotationId rotationId; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index a0d2630091b..27bcb4bd49a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.path.Path; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; @@ -26,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -35,6 +37,7 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import org.junit.Test; @@ -784,6 +787,30 @@ public class ControllerTest { } @Test + public void testDeployWithRoutingGeneratorEndpoints() { + var context = tester.newDeploymentContext(); + var applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .build(); + + var zones = Set.of(systemTest.zone(tester.controller().system()), + stagingTest.zone(tester.controller().system()), + ZoneId.from("prod", "us-west-1")); + for (var zone : zones) { + tester.controllerTester().serviceRegistry().routingGeneratorMock() + .putEndpoints(context.deploymentIdIn(zone), + List.of(new RoutingEndpoint("http://legacy-endpoint", "hostname", + false, "upstreamName"))); + } + // Defer load balancer provisioning in all environments so that routing controller uses routing generator + context.deferLoadBalancerProvisioningIn(zones.stream().map(ZoneId::environment).collect(Collectors.toSet())) + .submit(applicationPackage) + .deploy(); + } + + @Test public void testDeployWithGlobalEndpointsAndMultipleRoutingMethods() { var context = tester.newDeploymentContext(); var zone1 = ZoneId.from("prod", "us-west-1"); @@ -914,4 +941,36 @@ public class ControllerTest { .rotations().get(0).clusterId()); } + @Test + public void testReadableApplications() { + var db = new MockCuratorDb(); + var tester = new DeploymentTester(new ControllerTester(db)); + + // Create and deploy two applications + var app1 = tester.newDeploymentContext("t1", "a1", "default") + .submit() + .deploy(); + var app2 = tester.newDeploymentContext("t2", "a2", "default") + .submit() + .deploy(); + assertEquals(2, tester.applications().readable().size()); + + // Write invalid data to one application + db.curator().set(Path.fromString("/controller/v1/applications/" + app2.application().id().serialized()), + new byte[]{(byte) 0xDE, (byte) 0xAD}); + + // Can read the remaining readable + assertEquals(1, tester.applications().readable().size()); + + // Unconditionally reading all applications fails + try { + tester.applications().asList(); + fail("Expected exception"); + } catch (Exception ignored) { + } + + // Deployment for readable application still succeeds + app1.submit().deploy(); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index c9ad121276f..21b6e729e41 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ServiceInfo; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -177,6 +178,8 @@ public class InternalStepRunnerTest { .deferLoadBalancerProvisioningIn(testZone.environment()); tester.newDeploymentContext(app.instanceId()) .deferLoadBalancerProvisioningIn(stagingZone.environment()); + tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(app.testerId().id(), testZone), List.of()); + tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(new DeploymentId(app.instanceId(), stagingZone), List.of()); tester.runner().run(); tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 98178f2a19f..8863651e1b5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -21,6 +21,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportCons import com.yahoo.vespa.hosted.controller.api.integration.resource.MockTenantCost; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; @@ -40,6 +42,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ConfigServerMock configServerMock; private final MemoryNameService memoryNameService = new MemoryNameService(); private final MemoryGlobalRoutingService memoryGlobalRoutingService = new MemoryGlobalRoutingService(); + private final RoutingGeneratorMock routingGeneratorMock = new RoutingGeneratorMock(); private final MockMailer mockMailer = new MockMailer(); private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); private final MockMeteringClient mockMeteringClient = new MockMeteringClient(); @@ -89,6 +92,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override + public RoutingGenerator routingGenerator() { + return routingGeneratorMock; + } + + @Override public MockMailer mailer() { return mockMailer; } @@ -189,6 +197,10 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return memoryGlobalRoutingService; } + public RoutingGeneratorMock routingGeneratorMock() { + return routingGeneratorMock; + } + public MockContactRetriever contactRetrieverMock() { return mockContactRetriever; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 0bcf25dd4b8..136ed508a33 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -1,25 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.routing.RoutingId; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import java.net.URI; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; @@ -52,16 +51,9 @@ public class RotationRepositoryTest { .region("us-west-1") .build(); - private DeploymentTester tester; - private RotationRepository repository; - private DeploymentContext application; - - @Before - public void before() { - tester = new DeploymentTester(new ControllerTester(rotationsConfig)); - repository = tester.controller().routing().rotations(); - application = tester.newDeploymentContext("tenant1", "app1", "default"); - } + private final DeploymentTester tester = new DeploymentTester(new ControllerTester(rotationsConfig)); + private final RotationRepository repository = tester.controller().routing().rotations(); + private final DeploymentContext application = tester.newDeploymentContext("tenant1", "app1", "default"); @Test public void assigns_and_reuses_rotation() { @@ -71,23 +63,36 @@ public class RotationRepositoryTest { assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), - EndpointList.global(RoutingId.of(application.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) - .primary().get().url()); + tester.controller().routing().endpointsOf(application.instanceId()).primary().get().url()); try (RotationLock lock = repository.lock()) { List<AssignedRotation> rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), application.instance(), lock); assertSingleRotation(expected, rotations, repository); + assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3")), + application.instance().rotations().get(0).regions()); } // Submitting once more assigns same rotation - application.submit(applicationPackage); + application.submit(applicationPackage).deploy(); assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); + + // Adding region updates rotation + var applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("us-east-3") + .region("us-west-1") + .region("us-central-1") + .build(); + application.submit(applicationPackage).deploy(); + assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3"), + RegionName.from("us-central-1")), + application.instance().rotations().get(0).regions()); } @Test public void strips_whitespace_in_rotation_fqdn() { - tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces)); + var tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces)); RotationRepository repository = tester.controller().routing().rotations(); var application2 = tester.newDeploymentContext("tenant1", "app2", "default"); @@ -141,15 +146,19 @@ public class RotationRepositoryTest { public void prefixes_system_when_not_main() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .globalServiceId("foo") - .region("us-east-3") - .region("us-west-1") + .region("cd-us-central-1") + .region("cd-us-west-1") .build(); + var zones = List.of(ZoneApiMock.fromId("prod.cd-us-central-1"), ZoneApiMock.fromId("prod.cd-us-west-1")); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.shared) + .setSystemName(SystemName.cd); var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); application2.submit(applicationPackage); assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", - EndpointList.global(RoutingId.of(application2.instanceId(), EndpointId.defaultId()), SystemName.cd, RoutingMethod.shared) - .primary().get().url().toString()); + tester.controller().routing().endpointsOf(application2.instanceId()).primary().get().url().toString()); } @Test @@ -165,11 +174,9 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - EndpointList.global(RoutingId.of(instance1.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) - .primary().get().url()); + tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url()); assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - EndpointList.global(RoutingId.of(instance2.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) - .primary().get().url()); + tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url()); } @Test @@ -187,11 +194,9 @@ public class RotationRepositoryTest { assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - EndpointList.global(RoutingId.of(instance1.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) - .primary().get().url()); + tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url()); assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - EndpointList.global(RoutingId.of(instance2.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared) - .primary().get().url()); + tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url()); } private void assertSingleRotation(Rotation expected, List<AssignedRotation> assignedRotations, RotationRepository repository) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 719b2cc47f9..e44a1364185 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -24,8 +24,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; @@ -659,8 +659,12 @@ public class RoutingPoliciesTest { } private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { - var endpoint = EndpointList.global(RoutingId.of(application, endpointId), tester.controller().system(), RoutingMethod.exclusive) - .primary().get().dnsName(); + var endpoint = tester.controller().routing().endpointsOf(application) + .named(endpointId) + .targets(List.of(zone)) + .primary() + .map(Endpoint::dnsName) + .orElse("<none>"); var zoneTargets = Arrays.stream(zone) .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + z.value() + "/dns-zone-1/" + z.value()) |