Merge pull request #13589 from vespa-engine/mpolden/cfg-lb-dns

Maintain routing policies for system applications

10 files changed, 177 insertions, 10 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 463ffb58460..c441188b1be 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
 import com.yahoo.vespa.hosted.controller.application.EndpointList;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
 import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
@@ -84,13 +85,14 @@ public class RoutingController {
     /** Returns zone-scoped endpoints for given deployment */
     public EndpointList endpointsOf(DeploymentId deployment) {
         var endpoints = new LinkedHashSet<Endpoint>();
+        boolean isSystemApplication = SystemApplication.matching(deployment.applicationId()).isPresent();
         // Avoid reading application more than once per call to this
         var application = Suppliers.memoize(() -> controller.applications().requireApplication(TenantAndApplicationId.from(deployment.applicationId())));
         for (var policy : routingPolicies.get(deployment).values()) {
             if (!policy.status().isActive()) continue;
             for (var routingMethod :  controller.zoneRegistry().routingMethods(policy.id().zone())) {
-                if (routingMethod.isDirect() && !canRouteDirectlyTo(deployment, application.get())) continue;
-                endpoints.add(policy.endpointIn(controller.system(), routingMethod));
+                if (routingMethod.isDirect() && !isSystemApplication && !canRouteDirectlyTo(deployment, application.get())) continue;
+                endpoints.add(policy.endpointIn(controller.system(), routingMethod, controller.zoneRegistry()));
             }
         }
         return EndpointList.copyOf(endpoints);
@@ -98,6 +100,7 @@ public class RoutingController {
 
     /** Returns global-scoped endpoints for given instance */
     public EndpointList endpointsOf(ApplicationId instance) {
+        if (SystemApplication.matching(instance).isPresent()) return EndpointList.copyOf(List.of());
         return endpointsOf(controller.applications().requireApplication(TenantAndApplicationId.from(instance)),
                                                                         instance.instance());
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index ed5add8b98a..f8982c96637 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -36,20 +36,18 @@ public class Endpoint {
     private final RoutingMethod routingMethod;
     private final boolean tls;
 
-    private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system,
-                     Port port, boolean legacy, RoutingMethod routingMethod) {
+    private Endpoint(String name, URI url, List<ZoneId> zones, Scope scope, Port port, boolean legacy,
+                     RoutingMethod routingMethod) {
         Objects.requireNonNull(name, "name must be non-null");
-        Objects.requireNonNull(application, "application must be non-null");
         Objects.requireNonNull(zones, "zones must be non-null");
         Objects.requireNonNull(scope, "scope must be non-null");
-        Objects.requireNonNull(system, "system must be non-null");
         Objects.requireNonNull(port, "port must be non-null");
         Objects.requireNonNull(routingMethod, "routingMethod must be non-null");
         if (scope == Scope.zone && zones.size() != 1) {
             throw new IllegalArgumentException("A single zone must be given for zone-scoped endpoints");
         }
         this.name = name;
-        this.url = createUrl(name, application, zones, scope, system, port, legacy, routingMethod);
+        this.url = url;
         this.zones = List.copyOf(zones);
         this.scope = scope;
         this.legacy = legacy;
@@ -57,6 +55,20 @@ public class Endpoint {
         this.tls = port.tls;
     }
 
+    private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system,
+                     Port port, boolean legacy, RoutingMethod routingMethod) {
+        this(name,
+             createUrl(name,
+                       Objects.requireNonNull(application, "application must be non-null"),
+                       zones,
+                       scope,
+                       Objects.requireNonNull(system, "system must be non-null"),
+                       port,
+                       legacy,
+                       routingMethod),
+             zones, scope, port, legacy, routingMethod);
+    }
+
     /**
      * Returns the name of this endpoint (the first component of the DNS name). Depending on the endpoint type, this
      * can be one of the following:
@@ -286,6 +298,14 @@ public class Endpoint {
         return new EndpointBuilder(application);
     }
 
+    /** Create an endpoint for given system application */
+    public static Endpoint of(SystemApplication systemApplication, ZoneId zone, URI url) {
+        if (!systemApplication.hasEndpoint()) throw new IllegalArgumentException(systemApplication + " has no endpoint");
+        RoutingMethod routingMethod = RoutingMethod.exclusive;
+        Port port = url.getPort() == -1 ? Port.tls() : Port.tls(url.getPort()); // System application endpoints are always TLS
+        return new Endpoint("", url, List.of(zone), Scope.zone, port, false, routingMethod);
+    }
+
     public static class EndpointBuilder {
 
         private final ApplicationId application;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
index 243bca8c027..e1acf867744 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
@@ -8,7 +8,9 @@ import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
 
@@ -71,11 +73,27 @@ public enum SystemApplication {
         return nodeType.isDockerHost();
     }
 
+    /** Returns whether this has an endpoint */
+    public boolean hasEndpoint() {
+        return this == configServer;
+    }
+
+    /** Returns the endpoint of this, if any */
+    public Optional<Endpoint> endpointIn(ZoneId zone, ZoneRegistry zoneRegistry) {
+        if (!hasEndpoint()) return Optional.empty();
+        return Optional.of(Endpoint.of(this, zone, zoneRegistry.getConfigServerVipUri(zone)));
+    }
+
     /** All known system applications */
     public static List<SystemApplication> all() {
         return List.of(values());
     }
 
+    /** Returns the system application matching given id, if any */
+    public static Optional<SystemApplication> matching(ApplicationId id) {
+        return Arrays.stream(values()).filter(app -> app.id().equals(id)).findFirst();
+    }
+
     @Override
     public String toString() {
         return String.format("system application %s of type %s", id, nodeType);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 6f3e868dc1a..ca695a2d234 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -43,6 +43,7 @@ public class ControllerMaintenance extends AbstractComponent {
     private final CloudEventReporter cloudEventReporter;
     private final RotationStatusUpdater rotationStatusUpdater;
     private final ResourceTagMaintainer resourceTagMaintainer;
+    private final SystemRoutingPolicyMaintainer systemRoutingPolicyMaintainer;
 
     @Inject
     @SuppressWarnings("unused") // instantiated by Dependency Injection
@@ -71,6 +72,7 @@ public class ControllerMaintenance extends AbstractComponent {
         cloudEventReporter = new CloudEventReporter(controller, Duration.ofDays(1));
         rotationStatusUpdater = new RotationStatusUpdater(controller, maintenanceInterval);
         resourceTagMaintainer = new ResourceTagMaintainer(controller, Duration.ofMinutes(30), controller.serviceRegistry().resourceTagger());
+        systemRoutingPolicyMaintainer = new SystemRoutingPolicyMaintainer(controller, Duration.ofMinutes(10));
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -97,6 +99,7 @@ public class ControllerMaintenance extends AbstractComponent {
         cloudEventReporter.close();
         rotationStatusUpdater.close();
         resourceTagMaintainer.close();
+        systemRoutingPolicyMaintainer.close();
     }
 
     /** Create one OS upgrader per cloud found in the zone registry of controller */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java
new file mode 100644
index 00000000000..6c271ed0470
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java
@@ -0,0 +1,40 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
+
+import java.time.Duration;
+
+/**
+ * This maintains {@link RoutingPolicy}'s for {@link SystemApplication}s. In contrast to regular applications, this
+ * refreshes policies at an interval, not on deployment.
+ *
+ * @author mpolden
+ */
+public class SystemRoutingPolicyMaintainer extends ControllerMaintainer {
+
+    private final BooleanFlag featureFlag;
+
+    public SystemRoutingPolicyMaintainer(Controller controller, Duration interval) {
+        super(controller, interval);
+        this.featureFlag = Flags.CONFIGSERVER_PROVISION_LB.bindTo(controller.flagSource());
+    }
+
+    @Override
+    protected void maintain() {
+        for (var zone : controller().zoneRegistry().zones().all().ids()) {
+            for (var application : SystemApplication.values()) {
+                if (!application.hasEndpoint()) continue;
+                if (!featureFlag.with(FetchVector.Dimension.ZONE_ID, zone.value()).value()) continue;
+                controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone);
+            }
+        }
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index 7d7803915be..abd4770ea0d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -175,7 +175,8 @@ public class RoutingPolicies {
 
     /** Update zone DNS record for given policy */
     private void updateZoneDnsOf(RoutingPolicy policy) {
-        var name = RecordName.from(policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName());
+        var name = RecordName.from(policy.endpointIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry())
+                                         .dnsName());
         var data = RecordData.fqdn(policy.canonicalName().value());
         nameUpdaterIn(policy.id().zone()).createCname(name, data);
     }
@@ -190,7 +191,7 @@ public class RoutingPolicies {
             if (activeIds.contains(policy.id()) ||
                 !policy.id().zone().equals(allocation.deployment.zoneId())) continue;
 
-            var dnsName = policy.endpointIn(controller.system(), RoutingMethod.exclusive).dnsName();
+            var dnsName = policy.endpointIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry()).dnsName();
             nameUpdaterIn(allocation.deployment.zoneId()).removeRecords(Record.Type.CNAME, RecordName.from(dnsName));
             newPolicies.remove(policy.id());
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
index 37027e8a8c9..c56a5f0bd66 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
@@ -5,9 +5,11 @@ import com.google.common.collect.ImmutableSortedSet;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.RoutingMethod;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 
 import java.util.Objects;
 import java.util.Optional;
@@ -68,7 +70,10 @@ public class RoutingPolicy {
     }
 
     /** Returns the endpoint of this */
-    public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod) {
+    public Endpoint endpointIn(SystemName system, RoutingMethod routingMethod, ZoneRegistry zoneRegistry) {
+        Optional<Endpoint> infraEndpoint = SystemApplication.matching(id.owner())
+                                                            .flatMap(app -> app.endpointIn(id.zone(), zoneRegistry));
+        if (infraEndpoint.isPresent()) return infraEndpoint.get();
         return Endpoint.of(id.owner())
                        .target(id.cluster(), id.zone())
                        .on(Port.fromRoutingMethod(routingMethod))
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java
new file mode 100644
index 00000000000..8d6316d447f
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainerTest.java
@@ -0,0 +1,61 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+/**
+ * @author mpolden
+ */
+public class SystemRoutingPolicyMaintainerTest {
+
+    @Test
+    public void maintain() {
+        var tester = new ControllerTester();
+        var updater = new SystemRoutingPolicyMaintainer(tester.controller(), Duration.ofDays(1));
+        var dispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofDays(1), Integer.MAX_VALUE);
+
+        var zone = ZoneId.from("prod", "us-west-1");
+        tester.zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(zone));
+        tester.configServer().putLoadBalancers(zone, List.of(new LoadBalancer("lb1",
+                                                                              SystemApplication.configServer.id(),
+                                                                              ClusterSpec.Id.from("config"),
+                                                                              HostName.from("lb1.example.com"),
+                                                                              LoadBalancer.State.active,
+                                                                              Optional.of("dns-zone-1"))));
+
+        // Nothing happens without feature flag
+        updater.run();
+        dispatcher.run();
+        assertEquals(Set.of(), tester.nameService().records());
+
+        // Record is created
+        ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.CONFIGSERVER_PROVISION_LB.id(), true);
+        updater.run();
+        dispatcher.run();
+        Set<Record> records = tester.nameService().records();
+        assertEquals(1, records.size());
+        Record record = records.iterator().next();
+        assertSame(Record.Type.CNAME, record.type());
+        assertEquals("cfg.prod.us-west-1.test.vip", record.name().asString());
+        assertEquals("lb1.example.com.", record.data().asString());
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index ecb5c319f44..acd542b001c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -52,6 +52,9 @@
       "name": "RotationStatusUpdater"
     },
     {
+      "name": "SystemRoutingPolicyMaintainer"
+    },
+    {
       "name": "SystemUpgrader"
     },
     {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index 19e684aa175..56fb7194e4e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
 import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 import org.junit.Test;
 
@@ -579,6 +580,18 @@ public class RoutingPoliciesTest {
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
     }
 
+    @Test
+    public void config_server_routing_policy() {
+        var tester = new RoutingPoliciesTester();
+        var app = SystemApplication.configServer.id();
+
+        tester.provisionLoadBalancers(1, app, zone1);
+        tester.routingPolicies().refresh(app, DeploymentSpec.empty, zone1);
+        new NameServiceDispatcher(tester.tester.controller(), Duration.ofDays(1), Integer.MAX_VALUE).run();
+
+        assertEquals(Set.of("cfg.prod.us-west-1.test.vip"), tester.recordNames());
+    }
+
     /** Returns an application package builder that satisfies requirements for a directly routed endpoint */
     private static ApplicationPackageBuilder applicationPackageBuilder() {
         return new ApplicationPackageBuilder()