1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.maintenance;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.HostName;
import com.yahoo.log.LogLevel;
import com.yahoo.vespa.curator.Lock;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId;
import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.logging.Logger;
import java.util.stream.Collectors;
/**
* Maintains DNS records as defined by routing policies for all exclusive load balancers in this system.
*
* @author mortent
* @author mpolden
*/
public class RoutingPolicyMaintainer extends Maintainer {
private static final Logger log = Logger.getLogger(RoutingPolicyMaintainer.class.getName());
private final NameService nameService;
private final CuratorDb db;
public RoutingPolicyMaintainer(Controller controller,
Duration interval,
JobControl jobControl,
NameService nameService,
CuratorDb db) {
super(controller, interval, jobControl);
this.nameService = nameService;
this.db = db;
}
@Override
protected void maintain() {
Map<DeploymentId, List<LoadBalancer>> loadBalancers = findLoadBalancers();
updateDnsRecords(loadBalancers);
removeObsoleteDnsRecords(loadBalancers);
}
/** Find all exclusive load balancers owned by given applications, grouped by deployment */
private Map<DeploymentId, List<LoadBalancer>> findLoadBalancers() {
Map<DeploymentId, List<LoadBalancer>> result = new LinkedHashMap<>();
for (ZoneId zone : controller().zoneRegistry().zones().controllerUpgraded().ids()) {
List<LoadBalancer> loadBalancers = findLoadBalancersIn(zone);
for (LoadBalancer loadBalancer : loadBalancers) {
DeploymentId deployment = new DeploymentId(loadBalancer.application(), zone);
result.compute(deployment, (k, existing) -> {
if (existing == null) {
existing = new ArrayList<>();
}
existing.add(loadBalancer);
return existing;
});
}
}
return Collections.unmodifiableMap(result);
}
/** Create DNS records for all exclusive load balancers */
private void updateDnsRecords(Map<DeploymentId, List<LoadBalancer>> loadBalancers) {
for (Map.Entry<DeploymentId, List<LoadBalancer>> entry : loadBalancers.entrySet()) {
ApplicationId application = entry.getKey().applicationId();
ZoneId zone = entry.getKey().zoneId();
try (Lock lock = db.lockRoutingPolicies()) {
Set<RoutingPolicy> policies = new LinkedHashSet<>(db.readRoutingPolicies(application));
for (LoadBalancer loadBalancer : entry.getValue()) {
try {
policies.add(registerDnsAlias(application, zone, loadBalancer));
} catch (Exception e) {
log.log(LogLevel.WARNING, "Failed to create or update DNS record for load balancer " +
loadBalancer.hostname() + ". Retrying in " + maintenanceInterval(),
e);
}
}
db.writeRoutingPolicies(application, policies);
}
}
}
/** Register DNS alias for given load balancer */
private RoutingPolicy registerDnsAlias(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) {
HostName alias = HostName.from(RoutingPolicy.createAlias(loadBalancer.cluster(), application, zone));
RecordName name = RecordName.from(alias.value());
RecordData data = RecordData.fqdn(loadBalancer.hostname().value());
List<Record> existingRecords = nameService.findRecords(Record.Type.CNAME, name);
if (existingRecords.size() > 1) {
throw new IllegalStateException("Found more than 1 CNAME record for " + name.asString() + ": " + existingRecords);
}
Optional<Record> record = existingRecords.stream().findFirst();
RecordId id;
if (record.isPresent()) {
id = record.get().id();
nameService.updateRecord(id, data);
} else {
id = nameService.createCname(name, data);
}
return new RoutingPolicy(application, id.asString(), alias, loadBalancer.hostname(), loadBalancer.dnsZone(),
loadBalancer.rotations());
}
/** Find all load balancers in given zone */
private List<LoadBalancer> findLoadBalancersIn(ZoneId zone) {
try {
return controller().applications().configServer().getLoadBalancers(zone);
} catch (Exception e) {
log.log(LogLevel.WARNING,
String.format("Got exception fetching load balancers in zone: %s. Retrying in %s",
zone.value(), maintenanceInterval()), e);
}
return Collections.emptyList();
}
/** Remove all DNS records that point to non-existing load balancers */
private void removeObsoleteDnsRecords(Map<DeploymentId, List<LoadBalancer>> loadBalancers) {
try (Lock lock = db.lockRoutingPolicies()) {
List<RoutingPolicy> removalCandidates = new ArrayList<>(db.readRoutingPolicies());
Set<HostName> activeLoadBalancers = loadBalancers.values().stream()
.flatMap(Collection::stream)
.map(LoadBalancer::hostname)
.collect(Collectors.toSet());
// Remove any active load balancers
removalCandidates.removeIf(lb -> activeLoadBalancers.contains(lb.canonicalName()));
for (RoutingPolicy policy : removalCandidates) {
try {
nameService.removeRecord(new RecordId(policy.recordId()));
} catch (Exception e) {
log.log(LogLevel.WARNING, "Failed to remove DNS record with ID '" + policy.recordId() +
"'. Retrying in " + maintenanceInterval());
}
}
}
}
}
|
