diff options
author | jonmv <venstad@gmail.com> | 2023-02-01 16:27:48 +0100 |
---|---|---|
committer | jonmv <venstad@gmail.com> | 2023-02-01 16:27:48 +0100 |
commit | 4f76988e35d72006f29db1ba16e9610d70fe4864 (patch) | |
tree | 899ee2d768f690a755971a5e60ccd1ddb0266a3b /controller-server/src/main/java/com/yahoo | |
parent | f6fdb2f2a5385881af69de5e760c11bc4ddae7c1 (diff) |
Run DNS challenges asynchronously, and check from job runner
Diffstat (limited to 'controller-server/src/main/java/com/yahoo')
3 files changed, 83 insertions, 17 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 14f2b38f24a..9721396cd54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -358,8 +358,12 @@ public class InternalStepRunner implements StepRunner { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); Availability availability = endpointsAvailable(id.application(), id.type().zone(), logger); if (availability.status() == Status.available) { + if (controller.routing().policies().processDnsChallenges(new DeploymentId(id.application(), id.type().zone()))) { logger.log("Installation succeeded!"); return Optional.of(running); + } + logger.log("Waiting for DNS challenges for private endpoints to be processed"); + return Optional.empty(); } logger.log(availability.message()); if (availability.status() == Status.endpointsUnavailable && timedOut(id, deployment.get(), timeouts.endpoint())) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index be58370285a..f4980073d6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -15,12 +16,14 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.VespaChangeRequest; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; @@ -90,6 +93,7 @@ public class CuratorDb { private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); private static final Path routingPoliciesRoot = root.append("routingPolicies"); + private static final Path dnsChallengesRoot = root.append("dnsChallenges"); private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies"); private static final Path endpointCertificateRoot = root.append("applicationCertificates"); private static final Path archiveBucketsRoot = root.append("archiveBuckets"); @@ -114,6 +118,7 @@ public class CuratorDb { private final RunSerializer runSerializer = new RunSerializer(); private final RetriggerEntrySerializer retriggerEntrySerializer = new RetriggerEntrySerializer(); private final NotificationsSerializer notificationsSerializer = new NotificationsSerializer(); + private final DnsChallengeSerializer dnsChallengeSerializer = new DnsChallengeSerializer(); private final Curator curator; private final Duration tryLockTimeout; @@ -559,6 +564,35 @@ public class CuratorDb { .orElseGet(() -> new ZoneRoutingPolicy(zone, RoutingStatus.DEFAULT)); } + public void writeDnsChallenge(DnsChallenge challenge) { + curator.set(dnsChallengePath(challenge.clusterId()), dnsChallengeSerializer.toJson(challenge)); + } + + public void deleteDnsChallenge(ClusterId id) { + curator.delete(dnsChallengePath(id)); + } + + public List<DnsChallenge> readDnsChallenges(DeploymentId id) { + return curator.getChildren(dnsChallengePath(id)).stream() + .map(cluster -> readDnsChallenge(new ClusterId(id, ClusterSpec.Id.from(cluster)))) + .toList(); + } + + private DnsChallenge readDnsChallenge(ClusterId clusterId) { + return curator.getData(dnsChallengePath(clusterId)) + .map(bytes -> dnsChallengeSerializer.fromJson(bytes, clusterId)) + .orElseThrow(() -> new IllegalArgumentException("no DNS challenge for " + clusterId)); + } + + private static Path dnsChallengePath(DeploymentId id) { + return dnsChallengesRoot.append(id.applicationId().serializedForm()) + .append(id.zoneId().value()); + } + + private static Path dnsChallengePath(ClusterId id) { + return dnsChallengePath(id.deploymentId()).append(id.clusterId().value()); + } + // -------------- Application endpoint certificates ---------------------------- public void writeEndpointCertificateMetadata(ApplicationId applicationId, EndpointCertificateMetadata endpointCertificateMetadata) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 1c4916b9bed..61c71e964f6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -5,6 +5,7 @@ import ai.vespa.http.DomainName; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; @@ -20,6 +21,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.State; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedDirectTarget; import com.yahoo.vespa.hosted.controller.application.Endpoint; @@ -27,12 +30,14 @@ import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.yolean.UncheckedInterruptedException; import java.time.Instant; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -394,28 +399,51 @@ public class RoutingPolicies { new ClusterId(deploymentId, endpoint.cluster()), loadBalancer.cloudAccount()) .ifPresent(challenge -> { - try { + try (Mutex lock = db.lockNameServiceQueue()) { nameServiceForwarderIn(deploymentId.zoneId()).createTxt(challenge.name(), List.of(challenge.data()), Priority.high, ownerOf(deploymentId)); - Instant doom = controller.clock().instant().plusSeconds(30); - while (controller.clock().instant().isBefore(doom)) { - try (Mutex lock = controller.curator().lockNameServiceQueue()) { - if (controller.curator().readNameServiceQueue().requests().stream() - .noneMatch(request -> request.name().equals(challenge.name()))) { - try { challenge.trigger().run(); } - finally { nameServiceForwarderIn(deploymentId.zoneId()).removeRecords(Type.TXT, challenge.name(), Priority.normal, ownerOf(deploymentId)); } - return; - } - } - Thread.sleep(100); - } - throw new UncheckedTimeoutException("timed out waiting for DNS challenge to be processed"); - } - catch (InterruptedException e) { - throw new UncheckedInterruptedException("interrupted waiting for DNS challenge to be processed", e, true); + db.writeDnsChallenge(challenge); } }); } + /** Returns true iff. the given deployment has no incomplete DNS challenges, or throws (and cleans up) on errors. */ + public boolean processDnsChallenges(DeploymentId deploymentId) { + try (Mutex lock = db.lockNameServiceQueue()) { + List<DnsChallenge> challenges = new ArrayList<>(db.readDnsChallenges(deploymentId)); + Set<RecordName> pendingRequests = controller.curator().readNameServiceQueue().requests().stream() + .map(NameServiceRequest::name) + .collect(Collectors.toSet()); + try { + challenges.removeIf(challenge -> { + if (challenge.state() == State.pending) { + if (pendingRequests.contains(challenge.name())) return false; + challenge = challenge.withState(State.ready); + } + State state = controller.serviceRegistry().vpcEndpointService().process(challenge); + if (state == State.done) { + removeDnsChallenge(challenge); + return true; + } + else { + db.writeDnsChallenge(challenge.withState(state)); + return false; + } + }); + return challenges.isEmpty(); + } + catch (RuntimeException e) { + challenges.forEach(this::removeDnsChallenge); + throw e; + } + } + } + + private void removeDnsChallenge(DnsChallenge challenge) { + nameServiceForwarderIn(challenge.clusterId().deploymentId().zoneId()) + .removeRecords(Type.TXT, challenge.name(), Priority.normal, ownerOf(challenge.clusterId().deploymentId())); + db.deleteDnsChallenge(challenge.clusterId()); + } + /** * Remove policies and zone DNS records unreferenced by given load balancers * |