1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.maintenance;
import ai.vespa.metrics.ControllerMetrics;
import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
import com.yahoo.container.jdisc.secretstore.SecretStore;
import com.yahoo.jdisc.Metric;
import com.yahoo.transaction.Mutex;
import com.yahoo.vespa.flags.BooleanFlag;
import com.yahoo.vespa.flags.IntFlag;
import com.yahoo.vespa.flags.PermanentFlags;
import com.yahoo.vespa.flags.StringFlag;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate;
import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
/**
* Manages pool of ready-to-use randomized endpoint certificates
*
* @author andreer
*/
public class CertificatePoolMaintainer extends ControllerMaintainer {
private static final Logger log = Logger.getLogger(CertificatePoolMaintainer.class.getName());
private final CuratorDb curator;
private final SecretStore secretStore;
private final EndpointCertificateProvider endpointCertificateProvider;
private final Metric metric;
private final Controller controller;
private final IntFlag certPoolSize;
private final String dnsSuffix;
private final StringFlag endpointCertificateAlgo;
private final BooleanFlag useAlternateCertProvider;
public CertificatePoolMaintainer(Controller controller, Metric metric, Duration interval) {
super(controller, interval);
this.controller = controller;
this.secretStore = controller.secretStore();
this.certPoolSize = PermanentFlags.CERT_POOL_SIZE.bindTo(controller.flagSource());
this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource());
this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource());
this.curator = controller.curator();
this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider();
this.metric = metric;
this.dnsSuffix = Endpoint.dnsSuffix(controller.system());
}
protected double maintain() {
try {
moveRequestedCertsToReady();
List<UnassignedCertificate> certificatePool = curator.readUnassignedCertificates();
// Create metric for available certificates in the pool as a fraction of configured size
int poolSize = certPoolSize.value();
long available = certificatePool.stream().filter(c -> c.state() == UnassignedCertificate.State.ready).count();
metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of()));
if (certificatePool.size() < poolSize) {
provisionRandomizedCertificate();
}
} catch (Exception e) {
log.log(Level.SEVERE, "Exception caught while maintaining pool of unused randomized endpoint certs", e);
return 1.0;
}
return 0.0;
}
private void moveRequestedCertsToReady() {
try (Mutex lock = controller.curator().lockCertificatePool()) {
for (UnassignedCertificate cert : curator.readUnassignedCertificates()) {
if (cert.state() == UnassignedCertificate.State.ready) continue;
try {
OptionalInt maxKeyVersion = secretStore.listSecretVersions(cert.certificate().keyName()).stream().mapToInt(i -> i).max();
OptionalInt maxCertVersion = secretStore.listSecretVersions(cert.certificate().certName()).stream().mapToInt(i -> i).max();
if (maxKeyVersion.isPresent() && maxCertVersion.equals(maxKeyVersion)) {
curator.writeUnassignedCertificate(cert.withState(UnassignedCertificate.State.ready));
log.log(Level.INFO, "Randomized endpoint cert %s now ready for use".formatted(cert.id()));
}
} catch (SecretNotFoundException s) {
// Likely because the certificate is very recently provisioned - ignore till next time - should we log?
log.log(Level.INFO, "Could not yet read secrets for randomized endpoint cert %s - maybe next time ...".formatted(cert.id()));
}
}
}
}
private void provisionRandomizedCertificate() {
try (Mutex lock = controller.curator().lockCertificatePool()) {
Set<String> existingNames = controller.curator().readUnassignedCertificates().stream().map(UnassignedCertificate::id).collect(Collectors.toSet());
curator.readAssignedCertificates().stream()
.map(AssignedCertificate::certificate)
.map(EndpointCertificate::randomizedId)
.forEach(id -> id.ifPresent(existingNames::add));
String id = generateRandomId();
while (existingNames.contains(id)) id = generateRandomId();
EndpointCertificate f = endpointCertificateProvider.requestCaSignedCertificate(
"preprovisioned.%s".formatted(id),
List.of(
"*.%s.z%s".formatted(id, dnsSuffix),
"*.%s.g%s".formatted(id, dnsSuffix),
"*.%s.a%s".formatted(id, dnsSuffix)
),
Optional.empty(),
endpointCertificateAlgo.value(),
useAlternateCertProvider.value())
.withRandomizedId(id);
UnassignedCertificate certificate = new UnassignedCertificate(f, UnassignedCertificate.State.requested);
curator.writeUnassignedCertificate(certificate);
}
}
private String generateRandomId() {
return GeneratedEndpoint.createPart(controller.random(true));
}
}
|
