Merge pull request #16576 from vespa-engine/freva/expose-sync-uri

Expose node archive URL in nodes response

3 files changed, 104 insertions, 32 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 6582c03d727..aae64d0563c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -60,6 +60,7 @@ public class NodeRepository extends AbstractComponent {
     private final JobControl jobControl;
     private final Applications applications;
     private final LoadBalancers loadBalancers;
+    private final FlagSource flagSource;
     private final int spareCount;
 
     /**
@@ -102,11 +103,10 @@ public class NodeRepository extends AbstractComponent {
                           boolean useCuratorClientCache,
                           int spareCount,
                           long nodeCacheSize) {
-        // TODO (valerijf): Uncomment when exception for prod.cd-aws is removed
-//        if (provisionServiceProvider.getHostProvisioner().isPresent() != zone.getCloud().dynamicProvisioning())
-//            throw new IllegalArgumentException(String.format(
-//                    "dynamicProvisioning property must be 1-to-1 with availability of HostProvisioner, was: dynamicProvisioning=%s, hostProvisioner=%s",
-//                    zone.getCloud().dynamicProvisioning(), provisionServiceProvider.getHostProvisioner().map(__ -> "present").orElse("empty")));
+        if (provisionServiceProvider.getHostProvisioner().isPresent() != zone.getCloud().dynamicProvisioning())
+            throw new IllegalArgumentException(String.format(
+                    "dynamicProvisioning property must be 1-to-1 with availability of HostProvisioner, was: dynamicProvisioning=%s, hostProvisioner=%s",
+                    zone.getCloud().dynamicProvisioning(), provisionServiceProvider.getHostProvisioner().map(__ -> "present").orElse("empty")));
 
         this.db = new CuratorDatabaseClient(flavors, curator, clock, zone, useCuratorClientCache, nodeCacheSize);
         this.zone = zone;
@@ -122,6 +122,7 @@ public class NodeRepository extends AbstractComponent {
         this.jobControl = new JobControl(new JobControlFlags(db, flagSource));
         this.applications = new Applications(db);
         this.loadBalancers = new LoadBalancers(db);
+        this.flagSource = flagSource;
         this.spareCount = spareCount;
         rewriteNodes();
     }
@@ -174,6 +175,14 @@ public class NodeRepository extends AbstractComponent {
 
     public HostResourcesCalculator resourcesCalculator() { return resourcesCalculator; }
 
+    public FlagSource flagSource() { return flagSource; }
+
+    /** Returns the time keeper of this system */
+    public Clock clock() { return clock; }
+
+    /** Returns the zone of this system */
+    public Zone zone() { return zone; }
+
     /** The number of nodes we should ensure has free capacity for node failures whenever possible */
     public int spareCount() { return spareCount; }
 
@@ -204,10 +213,4 @@ public class NodeRepository extends AbstractComponent {
         applications.remove(transaction);
     }
 
-    /** Returns the time keeper of this system */
-    public Clock clock() { return clock; }
-
-    /** Returns the zone of this system */
-    public Zone zone() { return zone; }
-
 }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
index 502de3ef734..eaef044e07c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java
@@ -8,11 +8,12 @@ import com.yahoo.config.provision.Flavor;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.serialization.NetworkPortsSerializer;
 import com.yahoo.container.jdisc.HttpRequest;
-import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.restapi.SlimeJsonResponse;
 import com.yahoo.slime.Cursor;
-import com.yahoo.slime.JsonFormat;
-import com.yahoo.slime.Slime;
 import com.yahoo.vespa.applicationmodel.HostName;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Address;
@@ -22,8 +23,6 @@ import com.yahoo.vespa.orchestrator.Orchestrator;
 import com.yahoo.vespa.orchestrator.status.HostInfo;
 import com.yahoo.vespa.orchestrator.status.HostStatus;
 
-import java.io.IOException;
-import java.io.OutputStream;
 import java.net.URI;
 import java.util.List;
 import java.util.Optional;
@@ -33,7 +32,7 @@ import java.util.function.Function;
 /**
 * @author bratseth
 */
-class NodesResponse extends HttpResponse {
+class NodesResponse extends SlimeJsonResponse {
 
     /** The responses this can create */
     public enum ResponseType { nodeList, stateList, nodesInStateList, singleNode }
@@ -48,19 +47,16 @@ class NodesResponse extends HttpResponse {
     private final boolean recursive;
     private final Function<HostName, Optional<HostInfo>> orchestrator;
     private final NodeRepository nodeRepository;
-    private final Slime slime;
 
-    public NodesResponse(ResponseType responseType, HttpRequest request,  
+    public NodesResponse(ResponseType responseType, HttpRequest request,
                          Orchestrator orchestrator, NodeRepository nodeRepository) {
-        super(200);
         this.parentUrl = toParentUrl(request);
         this.nodeParentUrl = toNodeParentUrl(request);
-        filter = NodesV2ApiHandler.toNodeFilter(request);
+        this.filter = NodesV2ApiHandler.toNodeFilter(request);
         this.recursive = request.getBooleanProperty("recursive");
         this.orchestrator = orchestrator.getHostResolver();
         this.nodeRepository = nodeRepository;
 
-        slime = new Slime();
         Cursor root = slime.setObject();
         switch (responseType) {
             case nodeList: nodesToSlime(root); break;
@@ -84,16 +80,6 @@ class NodesResponse extends HttpResponse {
         return uri.getScheme() + "://" + uri.getHost() + ":" + uri.getPort() + "/nodes/v2/node/";
     }
 
-    @Override
-    public void render(OutputStream stream) throws IOException {
-        new JsonFormat(true).encode(stream, slime);
-    }
-
-    @Override
-    public String getContentType() {
-        return "application/json";
-    }
-
     private void statesToSlime(Cursor root) {
         Cursor states = root.setObject("states");
         for (Node.State state : Node.State.values())
@@ -193,6 +179,7 @@ class NodesResponse extends HttpResponse {
         node.reports().toSlime(object, "reports");
         node.modelName().ifPresent(modelName -> object.setString("modelName", modelName));
         node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname));
+        archiveUri(nodeRepository.flagSource(), node).ifPresent(uri -> object.setString("archiveUri", uri));
     }
 
     private void toSlime(ApplicationId id, Cursor object) {
@@ -247,4 +234,30 @@ class NodesResponse extends HttpResponse {
         return path.substring(lastSlash+1);
     }
 
+    // TODO (freva): Store this in Application or Node
+    static Optional<String> archiveUri(FlagSource flagSource, Node node) {
+        String bucket = Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.bindTo(flagSource)
+                .with(FetchVector.Dimension.NODE_TYPE, node.type().name())
+                .with(FetchVector.Dimension.APPLICATION_ID, node.allocation().map(alloc -> alloc.owner().serializedForm()).orElse(null))
+                .value();
+        if (bucket.isBlank()) return Optional.empty();
+
+        StringBuilder sb = new StringBuilder(100).append("s3://").append(bucket).append('/');
+        if (node.type() == NodeType.tenant) {
+            if (node.allocation().isEmpty()) return Optional.empty();
+            ApplicationId app = node.allocation().get().owner();
+
+            sb.append(app.tenant().value()).append('/').append(app.application().value()).append('/').append(app.instance().value()).append('/');
+        } else {
+            sb.append("hosted-vespa/");
+        }
+
+        for (char c: node.hostname().toCharArray()) {
+            if (c == '.') break;
+            sb.append(c);
+        }
+
+        return Optional.of(sb.append('/').toString());
+    }
+
 }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponseTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponseTest.java
new file mode 100644
index 00000000000..021afd0df7c
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponseTest.java
@@ -0,0 +1,56 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.restapi;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterMembership;
+import com.yahoo.config.provision.Flavor;
+import com.yahoo.config.provision.NodeResources;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.node.Allocation;
+import com.yahoo.vespa.hosted.provision.node.Generation;
+import org.junit.Test;
+
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author freva
+ */
+public class NodesResponseTest {
+    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
+
+    @Test
+    public void archive_uri() {
+        ApplicationId app = ApplicationId.from("vespa", "music", "main");
+        // Flag not set, no archive uri
+        assertArchiveUri(null, "h432a.prod.us-south-1.vespa.domain.tld", NodeType.tenant, app);
+        assertArchiveUri(null, "cfg1.prod.us-south-1.vespa.domain.tld", NodeType.config, app);
+
+        flagSource.withStringFlag(Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.id(), "vespa-data-bucket");
+        // Flag is set, but node not allocated, only sync non-tenant nodes
+        assertArchiveUri(null, "h432a.prod.us-south-1.vespa.domain.tld", NodeType.tenant, null);
+        assertArchiveUri("s3://vespa-data-bucket/hosted-vespa/cfg1/", "cfg1.prod.us-south-1.vespa.domain.tld", NodeType.config, null);
+
+        // Flag is set and node is allocated
+        assertArchiveUri("s3://vespa-data-bucket/vespa/music/main/h432a/", "h432a.prod.us-south-1.vespa.domain.tld", NodeType.tenant, app);
+        assertArchiveUri("s3://vespa-data-bucket/hosted-vespa/cfg1/", "cfg1.prod.us-south-1.vespa.domain.tld", NodeType.config, app);
+    }
+
+    private void assertArchiveUri(String archiveUri, String hostname, NodeType type, ApplicationId appId) {
+        Node.Builder nodeBuilder = Node.create("id", hostname, new Flavor(NodeResources.unspecified()), Node.State.parked, type);
+        Optional.ofNullable(appId)
+                .map(app -> new Allocation(app,
+                        ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()),
+                        NodeResources.unspecified(),
+                        Generation.initial(),
+                        false))
+                .ifPresent(nodeBuilder::allocation);
+
+        assertEquals(archiveUri, NodesResponse.archiveUri(flagSource, nodeBuilder.build()).orElse(null));
+    }
+}
\ No newline at end of file