Merge pull request #9760 from vespa-engine/mpolden/store-container-endpoints

Write container endpoints on prepare

7 files changed, 108 insertions, 12 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
index 542addbd7ff..4cabf39edcc 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
@@ -58,6 +58,9 @@ public final class PrepareParams {
         this.vespaVersion = vespaVersion;
         this.rotations = rotations;
         this.containerEndpoints = containerEndpoints;
+        if ((rotations != null && !rotations.isEmpty()) && !containerEndpoints.isEmpty()) {
+            throw new IllegalArgumentException("Cannot set both rotations and containerEndpoints");
+        }
     }
 
     public static class Builder {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index 0f24c3026d2..7af61a6efc1 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -8,18 +8,19 @@ import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
 import com.yahoo.config.application.api.ApplicationPackage;
 import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.OutOfCapacityException;
 import com.yahoo.config.provision.Rotation;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.lang.SettableOptional;
 import com.yahoo.log.LogLevel;
 import com.yahoo.path.Path;
+import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.config.server.ConfigServerSpec;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
 import com.yahoo.vespa.config.server.application.PermanentApplicationPackage;
@@ -30,6 +31,8 @@ import com.yahoo.vespa.config.server.http.InvalidApplicationException;
 import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
 import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
+import com.yahoo.vespa.config.server.tenant.ContainerEndpoint;
+import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache;
 import com.yahoo.vespa.config.server.tenant.Rotations;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.flags.FlagSource;
@@ -108,6 +111,10 @@ public class SessionPreparer {
             if ( ! params.isDryRun()) {
                 preparation.writeStateZK();
                 preparation.writeRotZK();
+                var globalServiceId = context.getApplicationPackage().getDeployment()
+                                             .map(DeploymentSpec::fromXml)
+                                             .flatMap(DeploymentSpec::globalServiceId);
+                preparation.writeContainerEndpointsZK(globalServiceId);
                 preparation.distribute();
             }
             log.log(LogLevel.DEBUG, () -> "time used " + params.getTimeoutBudget().timesUsed() +
@@ -132,7 +139,8 @@ public class SessionPreparer {
         /** The version of Vespa the application to be prepared specifies for its nodes */
         final com.yahoo.component.Version vespaVersion;
 
-        final Rotations rotations;
+        final Rotations rotations; // TODO: Remove this once we have migrated fully to container endpoints
+        final ContainerEndpointsCache containerEndpoints;
         final Set<Rotation> rotationsSet;
         final ModelContext.Properties properties;
 
@@ -153,6 +161,7 @@ public class SessionPreparer {
             this.applicationId = params.getApplicationId();
             this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion);
             this.rotations = new Rotations(curator, tenantPath);
+            this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator);
             this.rotationsSet = getRotations(params.rotations());
             this.properties = new ModelContextImpl.Properties(params.getApplicationId(),
                                                               configserverConfig.multitenant(),
@@ -225,6 +234,21 @@ public class SessionPreparer {
             checkTimeout("write rotations to zookeeper");
         }
 
+        void writeContainerEndpointsZK(Optional<String> globalServiceId) {
+            if (!params.containerEndpoints().isEmpty()) { // Use endpoints from parameter when explicitly given
+                containerEndpoints.write(applicationId, params.containerEndpoints());
+            } else { // Fall back to writing rotations as container endpoints
+                if (!rotationsSet.isEmpty()) {
+                    if (globalServiceId.isEmpty()) {
+                        log.log(LogLevel.WARNING, "Want to write rotations " + rotationsSet + " as container endpoints, but " + applicationId + " has no global-service-id. This should not happen");
+                        return;
+                    }
+                    containerEndpoints.write(applicationId, toContainerEndpoints(globalServiceId.get(), rotationsSet));
+                }
+            }
+            checkTimeout("write container endpoints to zookeeper");
+        }
+
         void distribute() {
             prepareResult.asList().forEach(modelResult -> modelResult.model
                                            .distributeFiles(modelResult.fileDistributionProvider.getFileDistribution()));
@@ -244,6 +268,13 @@ public class SessionPreparer {
 
     }
 
+    private static List<ContainerEndpoint> toContainerEndpoints(String globalServceId, Set<Rotation> rotations) {
+        return List.of(new ContainerEndpoint(new ClusterId(globalServceId),
+                                             rotations.stream()
+                                                      .map(Rotation::getId)
+                                                      .collect(Collectors.toUnmodifiableList())));
+    }
+
     private void writeStateToZooKeeper(SessionZooKeeperClient zooKeeperClient,
                                        ApplicationPackage applicationPackage,
                                        ApplicationId applicationId,
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpoint.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpoint.java
index eda320bde8f..b0fd3a81732 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpoint.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpoint.java
@@ -47,10 +47,7 @@ public class ContainerEndpoint {
 
     @Override
     public String toString() {
-        return "ContainerEndpoint{" +
-                "clusterId=" + clusterId +
-                ", names=" + names +
-                '}';
+        return String.format("container endpoint %s -> %s", clusterId, names);
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java
index 5e2cd5bd9d8..7e29f9abc1d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointsCache.java
@@ -28,7 +28,7 @@ public class ContainerEndpointsCache {
     private final Path cachePath;
     private final Curator curator;
 
-    ContainerEndpointsCache(Path tenantPath, Curator curator) {
+    public ContainerEndpointsCache(Path tenantPath, Curator curator) {
         this.cachePath = tenantPath.append("containerEndpointsCache/");
         this.curator = curator;
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
index 072dcf0e26f..6b2810af66c 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
@@ -1,22 +1,22 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.session;
 
+import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.Rotation;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.component.Version;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.LogLevel;
 import com.yahoo.path.Path;
 import com.yahoo.slime.Slime;
-import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.applicationmodel.ClusterId;
 import com.yahoo.vespa.config.server.MockReloadHandler;
-import com.yahoo.vespa.config.server.SuperModelGenerationCounter;
 import com.yahoo.vespa.config.server.TestComponentRegistry;
 import com.yahoo.vespa.config.server.TimeoutBudgetTest;
 import com.yahoo.vespa.config.server.application.PermanentApplicationPackage;
@@ -27,9 +27,10 @@ import com.yahoo.vespa.config.server.http.InvalidApplicationException;
 import com.yahoo.vespa.config.server.model.TestModelFactory;
 import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
+import com.yahoo.vespa.config.server.tenant.ContainerEndpoint;
+import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache;
 import com.yahoo.vespa.config.server.tenant.Rotations;
 import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
-
 import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import org.junit.Before;
@@ -42,6 +43,7 @@ import java.io.IOException;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 
@@ -49,8 +51,8 @@ import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.contains;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 /**
  * @author Ulf Lilleengen
@@ -170,6 +172,10 @@ public class SessionPreparerTest {
         assertThat(zkc.readApplicationId(), is(origId));
     }
 
+    private List<ContainerEndpoint> readContainerEndpoints(ApplicationId application) {
+        return new ContainerEndpointsCache(tenantPath, curator).read(application);
+    }
+
     private Set<Rotation> readRotationsFromZK(ApplicationId applicationId) {
         return new Rotations(curator, tenantPath).readRotationsFromZooKeeper(applicationId);
     }
@@ -205,6 +211,52 @@ public class SessionPreparerTest {
         assertThat(readRotationsFromZK(applicationId), contains(new Rotation(rotations)));
     }
 
+    @Test
+    public void require_that_rotations_are_written_as_container_endpoints() throws Exception {
+        var rotations = "app1.tenant1.global.vespa.example.com,rotation-042.vespa.global.routing";
+        var applicationId = applicationId("test");
+        var params = new PrepareParams.Builder().applicationId(applicationId).rotations(rotations).build();
+        prepare(new File("src/test/resources/deploy/hosted-app"), params);
+
+        var expected = List.of(new ContainerEndpoint(new ClusterId("qrs"),
+                                                     List.of("app1.tenant1.global.vespa.example.com",
+                                                             "rotation-042.vespa.global.routing")));
+        assertEquals(expected, readContainerEndpoints(applicationId));
+    }
+
+    @Test
+    public void require_that_container_endpoints_are_written() throws Exception {
+        var endpoints = "[\n" +
+                        "  {\n" +
+                        "    \"clusterId\": \"foo\",\n" +
+                        "    \"names\": [\n" +
+                        "      \"foo.app1.tenant1.global.vespa.example.com\",\n" +
+                        "      \"rotation-042.vespa.global.routing\"\n" +
+                        "    ]\n" +
+                        "  },\n" +
+                        "  {\n" +
+                        "    \"clusterId\": \"bar\",\n" +
+                        "    \"names\": [\n" +
+                        "      \"bar.app1.tenant1.global.vespa.example.com\",\n" +
+                        "      \"rotation-043.vespa.global.routing\"\n" +
+                        "    ]\n" +
+                        "  }\n" +
+                        "]";
+        var applicationId = applicationId("test");
+        var params = new PrepareParams.Builder().applicationId(applicationId)
+                                                .containerEndpoints(endpoints)
+                                                .build();
+        prepare(new File("src/test/resources/deploy/hosted-app"), params);
+
+        var expected = List.of(new ContainerEndpoint(new ClusterId("foo"),
+                                                     List.of("foo.app1.tenant1.global.vespa.example.com",
+                                                             "rotation-042.vespa.global.routing")),
+                               new ContainerEndpoint(new ClusterId("bar"),
+                                                     List.of("bar.app1.tenant1.global.vespa.example.com",
+                                                             "rotation-043.vespa.global.routing")));
+        assertEquals(expected, readContainerEndpoints(applicationId));
+    }
+
     private void prepare(File app) throws IOException {
         prepare(app, new PrepareParams.Builder().build());
     }
diff --git a/configserver/src/test/resources/deploy/hosted-app/deployment.xml b/configserver/src/test/resources/deploy/hosted-app/deployment.xml
new file mode 100644
index 00000000000..a92404c161d
--- /dev/null
+++ b/configserver/src/test/resources/deploy/hosted-app/deployment.xml
@@ -0,0 +1,7 @@
+<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<deployment version='1.0'>
+  <prod global-service-id="qrs">
+    <region active="true">us-north-1</region>
+    <region active="true">us-north-2</region>
+  </prod>
+</deployment>
diff --git a/configserver/src/test/resources/deploy/hosted-app/services.xml b/configserver/src/test/resources/deploy/hosted-app/services.xml
new file mode 100644
index 00000000000..57bee6ce9c9
--- /dev/null
+++ b/configserver/src/test/resources/deploy/hosted-app/services.xml
@@ -0,0 +1,6 @@
+<!-- Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
+<services version="1.0">
+  <container id="qrs" version="1.0">
+    <nodes count="2"/>
+  </container>
+</services>