diff options
Diffstat (limited to 'controller-server/src/main')
18 files changed, 804 insertions, 352 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index dfc9574fcd7..82120f13b75 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableList; @@ -59,7 +59,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.maintenance.RoutingPolicies; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; @@ -686,9 +686,9 @@ public class ApplicationController { catch (RuntimeException e) { log.log(Level.WARNING, "Failed to get endpoint information for " + id, e); } - return routingPolicies.get(id).stream() + return routingPolicies.get(id).values().stream() .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) - .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), + .collect(Collectors.toUnmodifiableMap(policy -> policy.id().cluster(), policy -> policy.endpointIn(controller.system()).url())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 4f6fe2ac2db..d3e21f0d399 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.ApplicationIdSnapshot; @@ -70,6 +71,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource private final FlagSource flagSource; private final NameServiceForwarder nameServiceForwarder; private final MavenRepository mavenRepository; + private final Metric metric; /** * Creates a controller @@ -77,22 +79,15 @@ public class Controller extends AbstractComponent implements ApplicationIdSource * @param curator the curator instance storing the persistent state of the controller. */ @Inject - public Controller(CuratorDb curator, RotationsConfig rotationsConfig, - AccessControl accessControl, - FlagSource flagSource, - MavenRepository mavenRepository, - ServiceRegistry serviceRegistry) { - this(curator, rotationsConfig, - accessControl, - com.yahoo.net.HostName::getLocalhost, flagSource, - mavenRepository, serviceRegistry); + public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, FlagSource flagSource, + MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric) { + this(curator, rotationsConfig, accessControl, com.yahoo.net.HostName::getLocalhost, flagSource, + mavenRepository, serviceRegistry, metric); } - public Controller(CuratorDb curator, RotationsConfig rotationsConfig, - AccessControl accessControl, - Supplier<String> hostnameSupplier, - FlagSource flagSource, MavenRepository mavenRepository, - ServiceRegistry serviceRegistry) { + public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, + Supplier<String> hostnameSupplier, FlagSource flagSource, MavenRepository mavenRepository, + ServiceRegistry serviceRegistry, Metric metric) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); @@ -101,7 +96,7 @@ public class Controller extends AbstractComponent implements ApplicationIdSource this.clock = Objects.requireNonNull(serviceRegistry.clock(), "Clock cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); - + this.metric = Objects.requireNonNull(metric, "Metric cannot be null"); metrics = new ConfigServerMetrics(serviceRegistry.configServer()); nameServiceForwarder = new NameServiceForwarder(curator); @@ -265,6 +260,10 @@ public class Controller extends AbstractComponent implements ApplicationIdSource return auditLogger; } + public Metric metric() { + return metric; + } + private Set<CloudName> clouds() { return zoneRegistry.zones().all().zones().stream() .map(ZoneApi::getCloudName) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index bd61d85fbc0..9d09394a571 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -17,6 +17,9 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -469,7 +472,7 @@ public class InternalStepRunner implements StepRunner { } private Optional<RunStatus> endTests(RunId id, DualLogger logger) { - if ( ! deployment(id.application(), id.type()).isPresent()) { + if (deployment(id.application(), id.type()).isEmpty()) { logger.log(INFO, "Deployment expired before tests could complete."); return Optional.of(aborted); } @@ -485,15 +488,22 @@ public class InternalStepRunner implements StepRunner { } } - Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); - if ( ! testerEndpoint.isPresent()) { - logger.log("Endpoints for tester not found -- trying again later."); - return Optional.empty(); - } - controller.jobController().updateTestLog(id); - TesterCloud.Status testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get()); + BooleanFlag useConfigServerForTesterAPI = Flags.USE_CONFIG_SERVER_FOR_TESTER_API_CALLS.bindTo(controller.flagSource()); + ZoneId zoneId = id.type().zone(controller.system()); + TesterCloud.Status testStatus; + if (useConfigServerForTesterAPI.with(FetchVector.Dimension.ZONE_ID, zoneId.value()).value()) { + testStatus = controller.serviceRegistry().configServer().getTesterStatus(new DeploymentId(id.application(), zoneId)); + } else { + Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id); + if (testerEndpoint.isEmpty()) { + logger.log("Endpoints for tester not found -- trying again later."); + return Optional.empty(); + } + testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get()); + } + switch (testStatus) { case NOT_STARTED: throw new IllegalStateException("Tester reports tests not started, even though they should have!"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 811daed256e..c8cfc8ac286 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -1,9 +1,10 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.jdisc.Metric; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -79,6 +80,7 @@ public class JobController { private final BufferedLogStore logs; private final TesterCloud cloud; private final Badges badges; + private final JobMetrics metric; private AtomicReference<Consumer<Run>> runner = new AtomicReference<>(__ -> { }); @@ -88,6 +90,7 @@ public class JobController { this.logs = new BufferedLogStore(curator, controller.serviceRegistry().runDataStore()); this.cloud = controller.serviceRegistry().testerCloud(); this.badges = new Badges(controller.zoneRegistry().badgeUrl()); + this.metric = new JobMetrics(controller.metric(), controller.system()); } public TesterCloud cloud() { return cloud; } @@ -360,6 +363,7 @@ public class JobController { } }); logs.flush(id); + metric.jobFinished(run.id().job(), finishedRun.status()); return finishedRun; }); } @@ -416,6 +420,7 @@ public class JobController { RunId newId = new RunId(id, type, last.map(run -> run.id().number()).orElse(0L) + 1); curator.writeLastRun(Run.initial(newId, versions, controller.clock().instant())); + metric.jobStarted(newId.job()); }); }); } @@ -526,7 +531,7 @@ public class JobController { DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); return controller.applications().getDeploymentEndpoints(testerId) .stream().findAny() - .or(() -> controller.applications().routingPolicies().get(testerId).stream() + .or(() -> controller.applications().routingPolicies().get(testerId).values().stream() .findAny() .map(policy -> policy.endpointIn(controller.system()).url())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java new file mode 100644 index 00000000000..a6ffb56492f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -0,0 +1,64 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; + +import java.util.Map; + +/** + * Records metrics related to deployment jobs. + * + * @author jonmv + */ +public class JobMetrics { + + public static final String start = "deployment.start"; + public static final String outOfCapacity = "deployment.outOfCapacity"; + public static final String deploymentFailure = "deployment.deploymentFailure"; + public static final String convergenceFailure = "deployment.convergenceFailure"; + public static final String testFailure = "deployment.testFailure"; + public static final String error = "deployment.error"; + public static final String abort = "deployment.abort"; + public static final String success = "deployment.success"; + + private final Metric metric; + private final SystemName system; + + public JobMetrics(Metric metric, SystemName system) { + this.metric = metric; + this.system = system; + } + + public void jobStarted(JobId id) { + metric.add(start, 1, metric.createContext(contextOf(id))); + } + + public void jobFinished(JobId id, RunStatus status) { + metric.add(valueOf(status), 1, metric.createContext(contextOf(id))); + } + + Map<String, String> contextOf(JobId id) { + return Map.of("tenant", id.application().tenant().value(), + "application", id.application().application().value(), + "instance", id.application().instance().value(), + "job", id.type().jobName(), + "environment", id.type().environment().value(), + "region", id.type().zone(system).region().value()); + } + + static String valueOf(RunStatus status) { + switch (status) { + case outOfCapacity: return outOfCapacity; + case deploymentFailed: return deploymentFailure; + case installationFailed: return convergenceFailure; + case testFailure: return testFailure; + case error: return error; + case aborted: return abort; + case success: return success; + default: throw new IllegalArgumentException("Unexpected run status '" + status + "'"); + } + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java deleted file mode 100644 index ee38b2c9516..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Updates routing policies and their associated DNS records based on an deployment's load balancers. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicies { - - private static final Logger LOGGER = Logger.getLogger(RoutingPolicies.class.getName()); - - private final Controller controller; - private final CuratorDb db; - - public RoutingPolicies(Controller controller) { - this.controller = Objects.requireNonNull(controller, "controller must be non-null"); - this.db = controller.curator(); - try (var lock = db.lockRoutingPolicies()) { // Update serialized format - for (var policy : db.readRoutingPolicies().entrySet()) { - db.writeRoutingPolicies(policy.getKey(), policy.getValue()); - } - } - } - - /** Read all known routing policies for given instance */ - public Set<RoutingPolicy> get(ApplicationId application) { - return db.readRoutingPolicies(application); - } - - /** Read all known routing policies for given deployment */ - public Set<RoutingPolicy> get(DeploymentId deployment) { - return get(deployment.applicationId(), deployment.zoneId()); - } - - /** Read all known routing policies for given deployment */ - public Set<RoutingPolicy> get(ApplicationId application, ZoneId zone) { - return db.readRoutingPolicies(application).stream() - .filter(policy -> policy.zone().equals(zone)) - .collect(Collectors.toUnmodifiableSet()); - } - - /** - * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if - * load balancers for given application have changed. - */ - public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { - if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; - var lbs = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer().getLoadBalancers(application, zone), - deploymentSpec); - try (var lock = db.lockRoutingPolicies()) { - removeObsoleteEndpointsFromDns(lbs, lock); - storePoliciesOf(lbs, lock); - removeObsoletePolicies(lbs, lock); - registerEndpointsInDns(lbs, lock); - } - } - - /** Create global endpoints for given route, if any */ - private void registerEndpointsInDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(get(loadBalancers.application)); - - // Create DNS record for each routing ID - for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { - Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().endpointId(), - controller.system()); - Set<AliasTarget> targets = routeEntry.getValue() - .stream() - .filter(policy -> policy.dnsZone().isPresent()) - .map(policy -> new AliasTarget(policy.canonicalName(), - policy.dnsZone().get(), - policy.zone())) - .collect(Collectors.toSet()); - controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); - } - } - - /** Store routing policies for given route. Returns the persisted policies. */ - private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = new LinkedHashSet<>(get(loadBalancers.application)); - for (LoadBalancer loadBalancer : loadBalancers.list) { - var endpointIds = loadBalancers.endpointIdsOf(loadBalancer); - var policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer, endpointIds); - if (!policies.add(policy)) { - // Update existing policy - policies.remove(policy); - policies.add(policy); - } - } - db.writeRoutingPolicies(loadBalancers.application, policies); - } - - /** Create a policy for given load balancer and register a CNAME for it */ - private RoutingPolicy createPolicy(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer, - Set<EndpointId> endpointIds) { - var routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, loadBalancer.hostname(), - loadBalancer.dnsZone(), endpointIds, isActive(loadBalancer)); - var name = RecordName.from(routingPolicy.endpointIn(controller.system()).dnsName()); - var data = RecordData.fqdn(loadBalancer.hostname().value()); - controller.nameServiceForwarder().createCname(name, data, Priority.normal); - return routingPolicy; - } - - /** Remove obsolete policies for given route and their CNAME records */ - private void removeObsoletePolicies(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var allPolicies = new LinkedHashSet<>(get(loadBalancers.application)); - var removalCandidates = new HashSet<>(allPolicies); - var activeLoadBalancers = loadBalancers.list.stream() - .map(LoadBalancer::hostname) - .collect(Collectors.toSet()); - // Remove active load balancers and irrelevant zones from candidates - removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) || - !policy.zone().equals(loadBalancers.zone)); - for (var policy : removalCandidates) { - var dnsName = policy.endpointIn(controller.system()).dnsName(); - controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); - allPolicies.remove(policy); - } - db.writeRoutingPolicies(loadBalancers.application, allPolicies); - } - - /** Remove unreferenced global endpoints for given route from DNS */ - private void removeObsoleteEndpointsFromDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var zonePolicies = get(loadBalancers.application, loadBalancers.zone); - var removalCandidates = routingTableFrom(zonePolicies).keySet(); - var activeRoutingIds = routingIdsFrom(loadBalancers); - removalCandidates.removeAll(activeRoutingIds); - for (var id : removalCandidates) { - var endpoint = RoutingPolicy.endpointOf(id.application(), id.endpointId(), controller.system()); - controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); - } - } - - /** Compute routing IDs from given load balancers */ - private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { - Set<RoutingId> routingIds = new LinkedHashSet<>(); - for (var loadBalancer : loadBalancers.list) { - for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { - routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); - } - } - return Collections.unmodifiableSet(routingIds); - } - - /** Compute a routing table from given policies */ - private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Set<RoutingPolicy> routingPolicies) { - var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); - for (var policy : routingPolicies) { - for (var rotation : policy.endpoints()) { - var id = new RoutingId(policy.owner(), rotation); - routingTable.putIfAbsent(id, new ArrayList<>()); - routingTable.get(id).add(policy); - } - } - return routingTable; - } - - private static boolean isActive(LoadBalancer loadBalancer) { - switch (loadBalancer.state()) { - case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early - // as possible - case active: return true; - } - return false; - } - - /** Load balancers allocated to a deployment */ - private static class AllocatedLoadBalancers { - - private final ApplicationId application; - private final ZoneId zone; - private final List<LoadBalancer> list; - private final DeploymentSpec deploymentSpec; - - private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, - DeploymentSpec deploymentSpec) { - this.application = application; - this.zone = zone; - this.list = List.copyOf(loadBalancers); - this.deploymentSpec = deploymentSpec; - } - - /** Compute all endpoint IDs for given load balancer */ - private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { - if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints - return Set.of(); - } - var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); - if (instanceSpec.isEmpty()) { - return Set.of(); - } - return instanceSpec.get().endpoints().stream() - .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) - .filter(endpoint -> endpoint.regions().contains(zone.region())) - .map(com.yahoo.config.application.api.Endpoint::endpointId) - .map(EndpointId::of) - .collect(Collectors.toSet()); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 22894a084b6..1a2ffc69249 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -18,19 +18,21 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import org.apache.zookeeper.data.Stat; import java.io.IOException; import java.io.UncheckedIOException; @@ -80,6 +82,7 @@ public class CuratorDb { private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); private static final Path routingPoliciesRoot = root.append("routingPolicies"); + private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies"); private static final Path applicationCertificateRoot = root.append("applicationCertificates"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); @@ -93,6 +96,7 @@ public class CuratorDb { private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer(); + private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); @@ -485,19 +489,28 @@ public class CuratorDb { // -------------- Routing policies ---------------------------------------- - public void writeRoutingPolicies(ApplicationId application, Set<RoutingPolicy> policies) { + public void writeRoutingPolicies(ApplicationId application, Map<RoutingPolicyId, RoutingPolicy> policies) { curator.set(routingPolicyPath(application), asJson(routingPolicySerializer.toSlime(policies))); } - public Map<ApplicationId, Set<RoutingPolicy>> readRoutingPolicies() { + public Map<ApplicationId, Map<RoutingPolicyId, RoutingPolicy>> readRoutingPolicies() { return curator.getChildren(routingPoliciesRoot).stream() .map(ApplicationId::fromSerializedForm) .collect(Collectors.toUnmodifiableMap(Function.identity(), this::readRoutingPolicies)); } - public Set<RoutingPolicy> readRoutingPolicies(ApplicationId application) { + public Map<RoutingPolicyId, RoutingPolicy> readRoutingPolicies(ApplicationId application) { return readSlime(routingPolicyPath(application)).map(slime -> routingPolicySerializer.fromSlime(application, slime)) - .orElseGet(Collections::emptySet); + .orElseGet(Map::of); + } + + public void writeZoneRoutingPolicy(ZoneRoutingPolicy policy) { + curator.set(zoneRoutingPolicyPath(policy.zone()), asJson(zoneRoutingPolicySerializer.toSlime(policy))); + } + + public ZoneRoutingPolicy readZoneRoutingPolicy(ZoneId zone) { + return readSlime(zoneRoutingPolicyPath(zone)).map(data -> zoneRoutingPolicySerializer.fromSlime(zone, data)) + .orElse(new ZoneRoutingPolicy(zone, GlobalRouting.DEFAULT_STATUS)); } // -------------- Application web certificates ---------------------------- @@ -581,6 +594,8 @@ public class CuratorDb { return routingPoliciesRoot.append(application.serializedForm()); } + private static Path zoneRoutingPolicyPath(ZoneId zone) { return zoneRoutingPoliciesRoot.append(zone.value()); } + private static Path nameServiceQueuePath() { return root.append("nameServiceQueue"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 54a3ef7551a..2429c5ee8c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; @@ -6,13 +6,20 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; +import java.time.Instant; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.Set; +import java.util.Map; /** * Serializer and deserializer for a {@link RoutingPolicy}. @@ -35,45 +42,64 @@ public class RoutingPolicySerializer { private static final String zoneField = "zone"; private static final String dnsZoneField = "dnsZone"; private static final String rotationsField = "rotations"; - private static final String activeField = "active"; + private static final String loadBalancerActiveField = "active"; + private static final String globalRoutingField = "globalRouting"; + private static final String agentField = "agent"; + private static final String changedAtField = "changedAt"; + private static final String statusField = "status"; - public Slime toSlime(Set<RoutingPolicy> routingPolicies) { + public Slime toSlime(Map<RoutingPolicyId, RoutingPolicy> routingPolicies) { var slime = new Slime(); var root = slime.setObject(); var policyArray = root.setArray(routingPoliciesField); - routingPolicies.forEach(policy -> { + routingPolicies.values().forEach(policy -> { var policyObject = policyArray.addObject(); - policyObject.setString(clusterField, policy.cluster().value()); - policyObject.setString(zoneField, policy.zone().value()); + policyObject.setString(clusterField, policy.id().cluster().value()); + policyObject.setString(zoneField, policy.id().zone().value()); policyObject.setString(canonicalNameField, policy.canonicalName().value()); policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone)); var rotationArray = policyObject.setArray(rotationsField); policy.endpoints().forEach(endpointId -> { rotationArray.addString(endpointId.id()); }); - policyObject.setBool(activeField, policy.active()); + policyObject.setBool(loadBalancerActiveField, policy.status().isActive()); + globalRoutingToSlime(policy.status().globalRouting(), policyObject.setObject(globalRoutingField)); }); return slime; } - public Set<RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) { - var policies = new LinkedHashSet<RoutingPolicy>(); + public Map<RoutingPolicyId, RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) { + var policies = new LinkedHashMap<RoutingPolicyId, RoutingPolicy>(); var root = slime.get(); var field = root.field(routingPoliciesField); field.traverse((ArrayTraverser) (i, inspect) -> { var endpointIds = new LinkedHashSet<EndpointId>(); inspect.field(rotationsField).traverse((ArrayTraverser) (j, endpointId) -> endpointIds.add(EndpointId.of(endpointId.asString()))); - var activeFieldInspector = inspect.field(activeField); - // TODO(mpolden): Remove field presence check after January 2020 - boolean active = !activeFieldInspector.valid() || activeFieldInspector.asBool(); - policies.add(new RoutingPolicy(owner, - ClusterSpec.Id.from(inspect.field(clusterField).asString()), - ZoneId.from(inspect.field(zoneField).asString()), - HostName.from(inspect.field(canonicalNameField).asString()), - Serializers.optionalString(inspect.field(dnsZoneField)), - endpointIds, active)); + var id = new RoutingPolicyId(owner, + ClusterSpec.Id.from(inspect.field(clusterField).asString()), + ZoneId.from(inspect.field(zoneField).asString())); + policies.put(id, new RoutingPolicy(id, + HostName.from(inspect.field(canonicalNameField).asString()), + Serializers.optionalString(inspect.field(dnsZoneField)), + endpointIds, + new Status(inspect.field(loadBalancerActiveField).asBool(), + globalRoutingFromSlime(inspect.field(globalRoutingField))))); }); - return Collections.unmodifiableSet(policies); + return Collections.unmodifiableMap(policies); + } + + public void globalRoutingToSlime(GlobalRouting globalRouting, Cursor object) { + object.setString(statusField, globalRouting.status().name()); + object.setString(agentField, globalRouting.agent().name()); + object.setLong(changedAtField, globalRouting.changedAt().toEpochMilli()); + } + + public GlobalRouting globalRoutingFromSlime(Inspector object) { + if (!object.valid()) return GlobalRouting.DEFAULT_STATUS; + var status = GlobalRouting.Status.valueOf(object.field(statusField).asString()); + var agent = GlobalRouting.Agent.valueOf(object.field(agentField).asString()); + var changedAt = Serializers.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH); + return new GlobalRouting(status, agent, changedAt); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java new file mode 100644 index 00000000000..6688d16ad14 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java @@ -0,0 +1,44 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; + +import java.util.Objects; + +/** + * Serializer for {@link ZoneRoutingPolicy}. + * + * @author mpolden + */ +public class ZoneRoutingPolicySerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String GLOBAL_ROUTING_FIELD = "globalRouting"; + + private final RoutingPolicySerializer routingPolicySerializer; + + public ZoneRoutingPolicySerializer(RoutingPolicySerializer routingPolicySerializer) { + this.routingPolicySerializer = Objects.requireNonNull(routingPolicySerializer, "routingPolicySerializer must be non-null"); + } + + public ZoneRoutingPolicy fromSlime(ZoneId zone, Slime slime) { + var root = slime.get(); + return new ZoneRoutingPolicy(zone, routingPolicySerializer.globalRoutingFromSlime(root.field(GLOBAL_ROUTING_FIELD))); + } + + public Slime toSlime(ZoneRoutingPolicy policy) { + var slime = new Slime(); + var root = slime.setObject(); + routingPolicySerializer.globalRoutingToSlime(policy.globalRouting(), root.setObject(GLOBAL_ROUTING_FIELD)); + return slime; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 378013b5e6d..f6cf776cbfa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.Signatures; @@ -68,7 +68,6 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; @@ -804,9 +803,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .forEach(globalEndpointUrls::add); // Per-cluster endpoints. These are backed by load balancers. - Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id()); + var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values(); for (var policy : routingPolicies) { - policy.rotationEndpointsIn(controller.system()).asList().stream() + policy.globalEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) .map(URI::toString) .forEach(globalEndpointUrls::add); @@ -929,10 +928,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .ifPresent(rotation -> object.setString("rotationId", rotation.asString())); // Per-cluster rotations - Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id()); - for (RoutingPolicy policy : routingPolicies) { - if (!policy.active()) continue; - policy.rotationEndpointsIn(controller.system()).asList().stream() + var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values(); + for (var policy : routingPolicies) { + if (!policy.status().isActive()) continue; + policy.globalEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) .map(URI::toString) .forEach(globalRotationsArray::addString); @@ -1043,11 +1042,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add endpoint(s) defined by routing policies var endpointArray = response.setArray("endpoints"); - for (var policy : controller.applications().routingPolicies().get(deploymentId)) { - if (!policy.active()) continue; + for (var policy : controller.applications().routingPolicies().get(deploymentId).values()) { + if (!policy.status().isActive()) continue; Cursor endpointObject = endpointArray.addObject(); Endpoint endpoint = policy.endpointIn(controller.system()); - endpointObject.setString("cluster", policy.cluster().value()); + endpointObject.setString("cluster", policy.id().cluster().value()); endpointObject.setBool("tls", endpoint.tls()); endpointObject.setString("url", endpoint.url().toString()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java new file mode 100644 index 00000000000..1b2cf4a7896 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java @@ -0,0 +1,85 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import java.time.Instant; +import java.util.Objects; + +/** + * Represents the global routing status of a {@link RoutingPolicy} or {@link ZoneRoutingPolicy}. This contains the + * time global routing status was last changed and who changed it. + * + * This is immutable. + * + * @author mpolden + */ +public class GlobalRouting { + + public static final GlobalRouting DEFAULT_STATUS = new GlobalRouting(Status.in, Agent.system, Instant.EPOCH); + + private final Status status; + private final Agent agent; + private final Instant changedAt; + + /** DO NOT USE. Public for serialization purposes */ + public GlobalRouting(Status status, Agent agent, Instant changedAt) { + this.status = Objects.requireNonNull(status, "status must be non-null"); + this.agent = Objects.requireNonNull(agent, "agent must be non-null"); + this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); + } + + /** The current status of this */ + public Status status() { + return status; + } + + /** The agent who last changed this */ + public Agent agent() { + return agent; + } + + /** The time this was last changed */ + public Instant changedAt() { + return changedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GlobalRouting that = (GlobalRouting) o; + return status == that.status && + agent == that.agent && + changedAt.equals(that.changedAt); + } + + @Override + public int hashCode() { + return Objects.hash(status, agent, changedAt); + } + + @Override + public String toString() { + return "status " + status + ", changed by " + agent + " @ " + changedAt; + } + + public static GlobalRouting status(Status status, Agent agent, Instant instant) { + return new GlobalRouting(status, agent, instant); + } + + // Used in serialization. Do not change. + public enum Status { + /** Status is determined by health checks **/ + in, + + /** Status is explicitly set to out */ + out, + } + + /** Agents that can change the state of global routing */ + public enum Agent { + operator, + tenant, + system, + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java index 7b0ec3d27ba..5543d0ea0b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java @@ -1,7 +1,8 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import java.util.Objects; @@ -42,4 +43,9 @@ public class RoutingId { return Objects.hash(application, endpointId); } + @Override + public String toString() { + return "routing id for " + endpointId + " of " + application; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java new file mode 100644 index 00000000000..c05152e7795 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -0,0 +1,288 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Updates routing policies and their associated DNS records based on an deployment's load balancers. + * + * @author mortent + * @author mpolden + */ +public class RoutingPolicies { + + private final Controller controller; + private final CuratorDb db; + + public RoutingPolicies(Controller controller) { + this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + this.db = controller.curator(); + try (var lock = db.lockRoutingPolicies()) { // Update serialized format + for (var policy : db.readRoutingPolicies().entrySet()) { + db.writeRoutingPolicies(policy.getKey(), policy.getValue()); + } + } + } + + /** Read all known routing policies for given instance */ + public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application) { + return db.readRoutingPolicies(application); + } + + /** Read all known routing policies for given deployment */ + public Map<RoutingPolicyId, RoutingPolicy> get(DeploymentId deployment) { + return get(deployment.applicationId(), deployment.zoneId()); + } + + /** Read all known routing policies for given deployment */ + public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application, ZoneId zone) { + return db.readRoutingPolicies(application).entrySet() + .stream() + .filter(kv -> kv.getKey().zone().equals(zone)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + /** + * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if + * load balancers for given application have changed. + */ + public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { + if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; + var loadBalancers = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer() + .getLoadBalancers(application, zone), + deploymentSpec); + var inactiveZones = inactiveZones(application, deploymentSpec); + try (var lock = db.lockRoutingPolicies()) { + removeGlobalDnsUnreferencedBy(loadBalancers, lock); + storePoliciesOf(loadBalancers, lock); + removePoliciesUnreferencedBy(loadBalancers, lock); + updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock); + } + } + + /** Set the status of all global endpoints in given zone */ + public void setGlobalRoutingStatus(ZoneId zone, GlobalRouting.Status status) { + try (var lock = db.lockRoutingPolicies()) { + db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator, + controller.clock().instant()))); + var allPolicies = db.readRoutingPolicies(); + for (var applicationPolicies : allPolicies.values()) { + updateGlobalDnsOf(applicationPolicies.values(), Set.of(), lock); + } + } + } + + /** Set the status of all global endpoints for given deployment */ + public void setGlobalRoutingStatus(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { + try (var lock = db.lockRoutingPolicies()) { + var policies = get(deployment.applicationId()); + var newPolicies = new LinkedHashMap<>(policies); + for (var policy : policies.values()) { + if (!policy.id().zone().equals(deployment.zoneId())) continue; // Wrong zone + var newPolicy = policy.with(policy.status().with(GlobalRouting.status(status, agent, + controller.clock().instant()))); + newPolicies.put(policy.id(), newPolicy); + } + db.writeRoutingPolicies(deployment.applicationId(), newPolicies); + updateGlobalDnsOf(newPolicies.values(), Set.of(), lock); + } + } + + /** Update global DNS record for given policies */ + private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + // Create DNS record for each routing ID + var routingTable = routingTableFrom(routingPolicies); + for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { + var targets = new LinkedHashSet<AliasTarget>(); + var staleTargets = new LinkedHashSet<AliasTarget>(); + for (var policy : routeEntry.getValue()) { + if (policy.dnsZone().isEmpty()) continue; + var target = new AliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone()); + var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); + // Remove target zone if global routing status is set out at: + // - zone level (ZoneRoutingPolicy) + // - deployment level (RoutingPolicy) + // - application package level (deployment.xml) + if (anyOut(zonePolicy.globalRouting(), policy.status().globalRouting()) || + inactiveZones.contains(policy.id().zone())) { + staleTargets.add(target); + } else { + targets.add(target); + } + } + if (!targets.isEmpty()) { + var endpoint = RoutingPolicy.globalEndpointOf(routeEntry.getKey().application(), + routeEntry.getKey().endpointId(), controller.system()); + controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); + } + staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, t.asData(), Priority.normal)); + } + } + + /** Store routing policies for given load balancers */ + private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var policies = new LinkedHashMap<>(get(loadBalancers.application)); + for (LoadBalancer loadBalancer : loadBalancers.list) { + var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.zone); + var existingPolicy = policies.get(policyId); + var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(), + loadBalancers.endpointIdsOf(loadBalancer), + new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS)); + // Preserve global routing status for existing policy + if (existingPolicy != null) { + newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().globalRouting())); + } + updateZoneDnsOf(newPolicy); + policies.put(newPolicy.id(), newPolicy); + } + db.writeRoutingPolicies(loadBalancers.application, policies); + } + + /** Update zone DNS record for given policy */ + private void updateZoneDnsOf(RoutingPolicy policy) { + var name = RecordName.from(policy.endpointIn(controller.system()).dnsName()); + var data = RecordData.fqdn(policy.canonicalName().value()); + controller.nameServiceForwarder().createCname(name, data, Priority.normal); + } + + /** Remove policies and zone DNS records unreferenced by given load balancers */ + private void removePoliciesUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var policies = get(loadBalancers.application); + var newPolicies = new LinkedHashMap<>(policies); + var activeLoadBalancers = loadBalancers.list.stream().map(LoadBalancer::hostname).collect(Collectors.toSet()); + for (var policy : policies.values()) { + // Leave active load balancers and irrelevant zones alone + if (activeLoadBalancers.contains(policy.canonicalName()) || + !policy.id().zone().equals(loadBalancers.zone)) continue; + + var dnsName = policy.endpointIn(controller.system()).dnsName(); + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); + newPolicies.remove(policy.id()); + } + db.writeRoutingPolicies(loadBalancers.application, newPolicies); + } + + /** Remove unreferenced global endpoints from DNS */ + private void removeGlobalDnsUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var zonePolicies = get(loadBalancers.application, loadBalancers.zone).values(); + var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet()); + var activeRoutingIds = routingIdsFrom(loadBalancers); + removalCandidates.removeAll(activeRoutingIds); + for (var id : removalCandidates) { + var endpoint = RoutingPolicy.globalEndpointOf(id.application(), id.endpointId(), controller.system()); + controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); + } + } + + /** Compute routing IDs from given load balancers */ + private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { + Set<RoutingId> routingIds = new LinkedHashSet<>(); + for (var loadBalancer : loadBalancers.list) { + for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { + routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); + } + } + return Collections.unmodifiableSet(routingIds); + } + + /** Compute a routing table from given policies */ + private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Collection<RoutingPolicy> routingPolicies) { + var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); + for (var policy : routingPolicies) { + for (var endpoint : policy.endpoints()) { + var id = new RoutingId(policy.id().owner(), endpoint); + routingTable.putIfAbsent(id, new ArrayList<>()); + routingTable.get(id).add(policy); + } + } + return Collections.unmodifiableMap(routingTable); + } + + private static boolean anyOut(GlobalRouting... globalRouting) { + return Arrays.stream(globalRouting) + .map(GlobalRouting::status) + .anyMatch(status -> status == GlobalRouting.Status.out); + } + + private static boolean isActive(LoadBalancer loadBalancer) { + switch (loadBalancer.state()) { + case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early + // as possible + case active: return true; + } + return false; + } + + /** Load balancers allocated to a deployment */ + private static class AllocatedLoadBalancers { + + private final ApplicationId application; + private final ZoneId zone; + private final List<LoadBalancer> list; + private final DeploymentSpec deploymentSpec; + + private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, + DeploymentSpec deploymentSpec) { + this.application = application; + this.zone = zone; + this.list = List.copyOf(loadBalancers); + this.deploymentSpec = deploymentSpec; + } + + /** Compute all endpoint IDs for given load balancer */ + private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { + if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints + return Set.of(); + } + var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); + if (instanceSpec.isEmpty()) { + return Set.of(); + } + return instanceSpec.get().endpoints().stream() + .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) + .filter(endpoint -> endpoint.regions().contains(zone.region())) + .map(com.yahoo.config.application.api.Endpoint::endpointId) + .map(EndpointId::of) + .collect(Collectors.toSet()); + } + + } + + /** Returns zones where global routing is declared inactive for instance through deploymentSpec */ + private static Set<ZoneId> inactiveZones(ApplicationId instance, DeploymentSpec deploymentSpec) { + var instanceSpec = deploymentSpec.instance(instance.instance()); + if (instanceSpec.isEmpty()) return Set.of(); + return instanceSpec.get().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .filter(zone -> !zone.active()) + .map(zone -> ZoneId.from(zone.environment(), zone.region().get())) + .collect(Collectors.toUnmodifiableSet()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 80a62d94f2e..b1b6d1ae58a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -1,60 +1,46 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import java.util.Objects; import java.util.Optional; import java.util.Set; /** - * Represents the DNS routing policy for a load balancer. A routing policy is uniquely identified by its owner, cluster - * and zone. + * Represents the DNS routing policy for a {@link com.yahoo.vespa.hosted.controller.application.Deployment}. * * @author mortent * @author mpolden */ public class RoutingPolicy { - private final ApplicationId owner; - private final ClusterSpec.Id cluster; - private final ZoneId zone; + private final RoutingPolicyId id; private final HostName canonicalName; private final Optional<String> dnsZone; private final Set<EndpointId> endpoints; - private final boolean active; + private final Status status; /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone, HostName canonicalName, - Optional<String> dnsZone, Set<EndpointId> endpoints, boolean active) { - this.owner = Objects.requireNonNull(owner, "owner must be non-null"); - this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); - this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + public RoutingPolicy(RoutingPolicyId id, HostName canonicalName, Optional<String> dnsZone, Set<EndpointId> endpoints, + Status status) { + this.id = Objects.requireNonNull(id, "id must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); this.endpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(endpoints, "endpoints must be non-null")); - this.active = active; + this.status = Objects.requireNonNull(status, "status must be non-null"); } - /** The application owning this */ - public ApplicationId owner() { - return owner; - } - - /** The zone this applies to */ - public ZoneId zone() { - return zone; - } - - /** The cluster this applies to */ - public ClusterSpec.Id cluster() { - return cluster; + /** The ID of this */ + public RoutingPolicyId id() { + return id; } /** The canonical name for this (rhs of a CNAME or ALIAS record) */ @@ -72,19 +58,24 @@ public class RoutingPolicy { return endpoints; } - /** Returns whether this is active (the underlying load balancer is in an active state) */ - public boolean active() { - return active; + /** Returns the status of this */ + public Status status() { + return status; + } + + /** Returns a copy of this with status set to given status */ + public RoutingPolicy with(Status status) { + return new RoutingPolicy(id, canonicalName, dnsZone, endpoints, status); } /** Returns the endpoint of this */ public Endpoint endpointIn(SystemName system) { - return Endpoint.of(owner).target(cluster, zone).on(Port.tls()).directRouting().in(system); + return Endpoint.of(id.owner()).target(id.cluster(), id.zone()).on(Port.tls()).directRouting().in(system); } - /** Returns rotation endpoints of this */ - public EndpointList rotationEndpointsIn(SystemName system) { - return EndpointList.of(endpoints.stream().map(endpointId -> endpointOf(owner, endpointId, system))); + /** Returns global endpoints which this is a member of */ + public EndpointList globalEndpointsIn(SystemName system) { + return EndpointList.of(endpoints.stream().map(endpointId -> globalEndpointOf(id.owner(), endpointId, system))); } @Override @@ -92,25 +83,23 @@ public class RoutingPolicy { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RoutingPolicy that = (RoutingPolicy) o; - return owner.equals(that.owner) && - cluster.equals(that.cluster) && - zone.equals(that.zone); + return id.equals(that.id); } @Override public int hashCode() { - return Objects.hash(owner, cluster, zone); + return Objects.hash(id); } @Override public String toString() { return String.format("%s [rotations: %s%s], %s owned by %s, in %s", canonicalName, endpoints, - dnsZone.map(z -> ", DNS zone: " + z).orElse(""), cluster, owner.toShortString(), - zone.value()); + dnsZone.map(z -> ", DNS zone: " + z).orElse(""), id.cluster(), id.owner().toShortString(), + id.zone().value()); } - /** Returns the endpoint of given rotation */ - public static Endpoint endpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { + /** Creates a global endpoint for given application */ + public static Endpoint globalEndpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java new file mode 100644 index 00000000000..06002e874f1 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java @@ -0,0 +1,57 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * Unique identifier for a {@link RoutingPolicy}. + * + * @author mpolden + */ +public class RoutingPolicyId { + + private final ApplicationId owner; + private final ClusterSpec.Id cluster; + private final ZoneId zone; + + public RoutingPolicyId(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone) { + this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); + this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + } + + /** The application owning this */ + public ApplicationId owner() { + return owner; + } + + /** The zone this applies to */ + public ZoneId zone() { + return zone; + } + + /** The cluster this applies to */ + public ClusterSpec.Id cluster() { + return cluster; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RoutingPolicyId that = (RoutingPolicyId) o; + return owner.equals(that.owner) && + cluster.equals(that.cluster) && + zone.equals(that.zone); + } + + @Override + public int hashCode() { + return Objects.hash(owner, cluster, zone); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java new file mode 100644 index 00000000000..51e59c7cf4f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java @@ -0,0 +1,53 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import java.util.Objects; + +/** + * Represents the status of a routing policy. + * + * This is immutable. + * + * @author mpolden + */ +public class Status { + + private final boolean active; + private final GlobalRouting globalRouting; + + /** DO NOT USE. Public for serialization purposes */ + public Status(boolean active, GlobalRouting globalRouting) { + this.active = active; + this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); + } + + /** Returns whether this is considered active according to the load balancer status */ + public boolean isActive() { + return active; + } + + /** Return status of global routing */ + public GlobalRouting globalRouting() { + return globalRouting; + } + + /** Returns a copy of this with global routing changed */ + public Status with(GlobalRouting globalRouting) { + return new Status(active, globalRouting); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Status status = (Status) o; + return active == status.active && + globalRouting.equals(status.globalRouting); + } + + @Override + public int hashCode() { + return Objects.hash(active, globalRouting); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java new file mode 100644 index 00000000000..262cacd325e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java @@ -0,0 +1,49 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * Represents the DNS routing policy for a zone. This takes precedence over of an individual {@link RoutingPolicy}. + * + * This is immutable. + * + * @author mpolden + */ +public class ZoneRoutingPolicy { + + private final ZoneId zone; + private final GlobalRouting globalRouting; + + public ZoneRoutingPolicy(ZoneId zone, GlobalRouting globalRouting) { + this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); + } + + /** The zone this applies to */ + public ZoneId zone() { + return zone; + } + + /** The status of global routing */ + public GlobalRouting globalRouting() { + return globalRouting; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ZoneRoutingPolicy that = (ZoneRoutingPolicy) o; + return zone.equals(that.zone) && + globalRouting.equals(that.globalRouting); + } + + @Override + public int hashCode() { + return Objects.hash(zone, globalRouting); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index f722eb4f6bb..7c3c30738d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -1,17 +1,13 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.InstanceList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import java.time.Instant; import java.time.ZoneOffset; -import java.util.stream.Collectors; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; |