Merge pull request #19955 from vespa-engine/mortent/lbservices-config

Generate routing endpoints in config model

8 files changed, 143 insertions, 7 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 5183a3ca587..49c968a1d91 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -72,6 +72,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private double diskBloatFactor = 0.2;
     private boolean distributorEnhancedMaintenanceScheduling = false;
     private boolean asyncApplyBucketDiff = false;
+    private List<String> zoneDnsSuffixes = List.of();
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -124,6 +125,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; }
     @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; }
     @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; }
+    @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; }
 
     public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) {
         this.maxUnCommittedMemory = maxUnCommittedMemory;
@@ -320,6 +322,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setZoneDnsSuffixes(List<String> zoneDnsSuffixes) {
+        this.zoneDnsSuffixes = List.copyOf(zoneDnsSuffixes);
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
index d6c66a635d4..f584b4cd207 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java
@@ -18,6 +18,7 @@ import com.yahoo.config.model.ApplicationConfigProducerRoot;
 import com.yahoo.config.model.ConfigModelRegistry;
 import com.yahoo.config.model.ConfigModelRepo;
 import com.yahoo.config.model.NullConfigModelRegistry;
+import com.yahoo.config.model.api.ApplicationClusterInfo;
 import com.yahoo.config.model.api.HostInfo;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.Provisioned;
@@ -696,4 +697,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri
                                       .collect(Collectors.toSet());
     }
 
+    @Override
+    public Set<ApplicationClusterInfo> applicationClusterInfo() {
+        return Set.copyOf(getContainerClusters().values());
+    }
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index c3897f49c44..a5dc26e19e3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -7,10 +7,14 @@ import com.yahoo.component.ComponentId;
 import com.yahoo.component.ComponentSpecification;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.ComponentInfo;
+import com.yahoo.config.model.api.ApplicationClusterEndpoint;
+import com.yahoo.config.model.api.ApplicationClusterInfo;
+import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.producer.AbstractConfigProducer;
 import com.yahoo.config.provision.AllocatedHosts;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostSpec;
 import com.yahoo.container.bundle.BundleInstantiationSpecification;
 import com.yahoo.container.di.config.ApplicationBundlesConfig;
@@ -26,6 +30,7 @@ import com.yahoo.vespa.config.search.RankProfilesConfig;
 import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
 import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
 import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
+import com.yahoo.vespa.model.AbstractService;
 import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainer;
 import com.yahoo.vespa.model.container.component.BindingPattern;
 import com.yahoo.vespa.model.container.component.Component;
@@ -39,6 +44,7 @@ import com.yahoo.vespa.model.utils.FileSender;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -60,7 +66,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
         ServletPathsConfig.Producer,
         ContainerMbusConfig.Producer,
         MetricsProxyApiConfig.Producer,
-        ZookeeperServerConfig.Producer {
+        ZookeeperServerConfig.Producer,
+        ApplicationClusterInfo {
 
     public static final String METRICS_V2_HANDLER_CLASS = MetricsV2Handler.class.getName();
     public static final BindingPattern METRICS_V2_HANDLER_BINDING_1 = SystemBindingPattern.fromHttpPath(MetricsV2Handler.V2_PATH);
@@ -88,6 +95,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
 
     private Integer memoryPercentage = null;
 
+    private List<ApplicationClusterEndpoint> endpointList = List.of();
+
     public ApplicationContainerCluster(AbstractConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) {
         super(parent, configSubId, clusterId, deployState, true);
         this.tlsClientAuthority = deployState.tlsClientAuthority();
@@ -115,6 +124,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
     protected void doPrepare(DeployState deployState) {
         addAndSendApplicationBundles(deployState);
         sendUserConfiguredFiles(deployState);
+        createEndpointList(deployState);
     }
 
     private void addAndSendApplicationBundles(DeployState deployState) {
@@ -184,6 +194,60 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
      */
     public Optional<Integer> getMemoryPercentage() { return Optional.ofNullable(memoryPercentage); }
 
+    /*
+      Create list of endpoints, these will be consumed later by the LBservicesProducer
+     */
+    private void createEndpointList(DeployState deployState) {
+        if(!deployState.isHosted()) return;
+        if(deployState.getProperties().applicationId().instance().isTester()) return;
+        List<ApplicationClusterEndpoint> endpoints = new ArrayList<>();
+        // Add zone local endpoints using zone dns suffixes, tenant, application and cluster id.
+        // For now support both L7 and L4 routing
+
+        List<String> hosts = getContainers().stream()
+                .map(AbstractService::getHostName)
+                .collect(Collectors.toList());
+        for(String suffix : deployState.getProperties().zoneDnsSuffixes()) {
+            // L4
+            ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom(
+                    ClusterSpec.Id.from(getName()),
+                    deployState.getProperties().applicationId(),
+                    suffix);
+            endpoints.add(ApplicationClusterEndpoint.builder()
+                                  .zoneScope()
+                                  .sharedL4Routing()
+                                  .dnsName(l4Name)
+                                  .hosts(hosts)
+                                  .build());
+
+            // L7
+            ApplicationClusterEndpoint.DnsName l7Name = ApplicationClusterEndpoint.DnsName.sharedNameFrom(
+                    ClusterSpec.Id.from(getName()),
+                    deployState.getProperties().applicationId(),
+                    suffix);
+            endpoints.add(ApplicationClusterEndpoint.builder()
+                                  .zoneScope()
+                                  .sharedRouting()
+                                  .dnsName(l7Name)
+                                  .hosts(hosts)
+                                  .build());
+        }
+
+        // Then get all endpoints provided by controller. Can be created with L4 routing only
+        Set<ContainerEndpoint> endpointsFromController = deployState.getEndpoints();
+        endpointsFromController.stream()
+                .filter(ce -> ce.clusterId().equals(getName()))
+                .forEach(ce -> ce.names().forEach(
+                        name -> endpoints.add(ApplicationClusterEndpoint.builder()
+                                                      .scope(ce.scope())
+                                                      .sharedL4Routing()
+                                                      .dnsName(ApplicationClusterEndpoint.DnsName.from(name))
+                                                      .hosts(hosts)
+                                                      .build())
+                ));
+        endpointList = List.copyOf(endpoints);
+    }
+
     @Override
     public void getConfig(ApplicationBundlesConfig.Builder builder) {
         applicationBundles.stream().map(FileReference::value)
@@ -293,6 +357,11 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
                         null))));
     }
 
+    @Override
+    public List<ApplicationClusterEndpoint> endpoints() {
+        return endpointList;
+    }
+
     public static class MbusParams {
         // the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%)
         final Double maxConcurrentFactor;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index 39d4d7ec6c8..ca4dccbbbe1 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -8,8 +8,10 @@ import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.Endpoint;
 import com.yahoo.config.model.ConfigModelContext;
 import com.yahoo.config.model.ConfigModelContext.ApplicationType;
+import com.yahoo.config.model.api.ApplicationClusterEndpoint;
 import com.yahoo.config.model.api.ConfigServerSpec;
 import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.EndpointCertificateSecrets;
@@ -338,9 +340,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
 
     private void setRotations(Container container, Set<ContainerEndpoint> endpoints, String containerClusterName) {
         var rotationsProperty = endpoints.stream()
-                                         .filter(endpoint -> endpoint.clusterId().equals(containerClusterName))
-                                         .flatMap(endpoint -> endpoint.names().stream())
-                                         .collect(Collectors.toUnmodifiableSet());
+                .filter(endpoint -> endpoint.clusterId().equals(containerClusterName))
+                // Only consider global endpoints.
+                .filter(endpoint -> endpoint.scope() == ApplicationClusterEndpoint.Scope.global)
+                .flatMap(endpoint -> endpoint.names().stream())
+                .collect(Collectors.toUnmodifiableSet());
 
         // Build the comma delimited list of endpoints this container should be known as.
         // Confusingly called 'rotations' for legacy reasons.
diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java b/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java
index 36da5c2f142..031f25306d5 100644
--- a/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/deploy/DeployStateTest.java
@@ -2,6 +2,7 @@
 package com.yahoo.config.model.deploy;
 
 import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.api.ApplicationClusterEndpoint;
 import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.HostProvisioner;
@@ -111,7 +112,7 @@ public class DeployStateTest {
     @Test
     public void testContainerEndpoints() {
         assertTrue(new DeployState.Builder().endpoints(Set.of()).build().getEndpoints().isEmpty());
-        var endpoints = Set.of(new ContainerEndpoint("c1", List.of("c1.example.com", "c1-alias.example.com")));
+        var endpoints = Set.of(new ContainerEndpoint("c1", ApplicationClusterEndpoint.Scope.global, List.of("c1.example.com", "c1-alias.example.com")));
         assertEquals(endpoints, new DeployState.Builder().endpoints(endpoints).build().getEndpoints());
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index 7190a7d4db1..2016cea02a9 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -7,6 +7,8 @@ import com.yahoo.cloud.config.CuratorConfig;
 import com.yahoo.cloud.config.ZookeeperServerConfig;
 import com.yahoo.component.ComponentId;
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.api.ApplicationClusterEndpoint;
+import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.deploy.DeployState;
 import com.yahoo.config.model.deploy.TestProperties;
 import com.yahoo.config.model.test.MockApplicationPackage;
@@ -36,8 +38,11 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 import java.util.stream.Collectors;
 
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasKey;
 import static org.junit.Assert.assertEquals;
@@ -351,6 +356,49 @@ public class ContainerClusterTest {
 
     }
 
+    @Test
+    public void generatesCorrectRoutingInfo() {
+
+        assertNames(ApplicationId.from("t1", "a1", "i1"),
+                    Set.of(),
+                    List.of("search-cluster.i1.a1.t1.endpoint.suffix", "search-cluster--i1--a1--t1.endpoint.suffix"));
+
+        assertNames(ApplicationId.from("t1", "a1", "default"),
+                    Set.of(),
+                    List.of("search-cluster.a1.t1.endpoint.suffix", "search-cluster--a1--t1.endpoint.suffix"));
+
+        assertNames(ApplicationId.from("t1", "default", "default"),
+                    Set.of(),
+                    List.of("search-cluster.default.t1.endpoint.suffix", "search-cluster--default--t1.endpoint.suffix"));
+
+        assertNames(ApplicationId.from("t1", "a1", "default"),
+                    Set.of(new ContainerEndpoint("not-in-this-cluster", ApplicationClusterEndpoint.Scope.global, List.of("foo", "bar"))),
+                    List.of("search-cluster.a1.t1.endpoint.suffix", "search-cluster--a1--t1.endpoint.suffix"));
+
+        assertNames(ApplicationId.from("t1", "a1", "default"),
+                    Set.of(new ContainerEndpoint("search-cluster", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z")),
+                           new ContainerEndpoint("search-cluster", ApplicationClusterEndpoint.Scope.application, List.of("app-rotation.x.y.z"))),
+                    List.of("search-cluster.a1.t1.endpoint.suffix", "search-cluster--a1--t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z"));
+    }
+
+    private void assertNames(ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedNames) {
+        DeployState state = new DeployState.Builder()
+                .zone(Zone.defaultZone())
+                .endpoints(globalEndpoints)
+                .properties(new TestProperties()
+                                    .setHostedVespa(true)
+                                    .setApplicationId(appId)
+                                    .setZoneDnsSuffixes(List.of(".endpoint.suffix")))
+                .build();
+        MockRoot root = new MockRoot("foo", state);
+        ApplicationContainerCluster cluster = new ApplicationContainerCluster(root, "container", "search-cluster", state);
+        addContainer(root, cluster, "c1", "host-c1");
+        cluster.doPrepare(state);
+        List<ApplicationClusterEndpoint> endpoints = cluster.endpoints();
+        assertEquals(expectedNames.size(), endpoints.size());
+        expectedNames.forEach(expected -> assertTrue("Endpoint not matched " + expected, endpoints.stream().anyMatch(e -> Objects.equals(e.dnsName().value(), expected))));
+    }
+
     private void verifyTesterApplicationInstalledBundles(Zone zone, List<String> expectedBundleNames) {
         ApplicationId appId = ApplicationId.from("tenant", "application", "instance-t");
         DeployState state = new DeployState.Builder().properties(
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java
index 6d51c3a50b6..1d774526a9b 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/http/BlockFeedGlobalEndpointsFilterTest.java
@@ -2,6 +2,7 @@
 
 package com.yahoo.vespa.model.container.http;
 
+import com.yahoo.config.model.api.ApplicationClusterEndpoint;
 import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.jdisc.http.filter.security.rule.RuleBasedFilterConfig;
 import org.hamcrest.Matchers;
@@ -19,7 +20,7 @@ public class BlockFeedGlobalEndpointsFilterTest {
 
     @Test
     public void setup_blocking_rule_when_endpoints_is_non_empty() {
-        var endpoints = Set.of(new ContainerEndpoint("default", List.of("foo", "bar")));
+        var endpoints = Set.of(new ContainerEndpoint("default", ApplicationClusterEndpoint.Scope.global, List.of("foo", "bar")));
         var filter = new BlockFeedGlobalEndpointsFilter(endpoints, true);
         var config = getConfig(filter);
         assertEquals(1, config.rule().size());
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
index be6d8ca5d0a..bd0d62c70a7 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java
@@ -5,6 +5,7 @@ import com.yahoo.cloud.config.ZookeeperServerConfig;
 import com.yahoo.component.ComponentId;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.model.NullConfigModelRegistry;
+import com.yahoo.config.model.api.ApplicationClusterEndpoint;
 import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.EndpointCertificateSecrets;
 import com.yahoo.config.model.api.ModelContext;
@@ -613,7 +614,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase {
         final var deployState = new DeployState.Builder()
                 .applicationPackage(applicationPackage)
                 .zone(new Zone(Environment.prod, RegionName.from("us-east-1")))
-                .endpoints(Set.of(new ContainerEndpoint("comics-search", List.of("nalle", "balle"))))
+                .endpoints(Set.of(new ContainerEndpoint("comics-search", ApplicationClusterEndpoint.Scope.global, List.of("nalle", "balle"))))
                 .properties(new TestProperties().setHostedVespa(true))
                 .build();