Use archiveUri from node-repo

7 files changed, 69 insertions, 98 deletions

diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
index c6f540aafb6..01a6e87fb58 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize;
 
+import java.net.URI;
 import java.time.Instant;
 import java.util.EnumSet;
 import java.util.Objects;
@@ -59,6 +60,7 @@ public class NodeSpec {
     private final NodeReports reports;
 
     private final Optional<String> parentHostname;
+    private final Optional<URI> archiveUri;
 
     public NodeSpec(
             String hostname,
@@ -85,7 +87,8 @@ public class NodeSpec {
             Set<String> ipAddresses,
             Set<String> additionalIpAddresses,
             NodeReports reports,
-            Optional<String> parentHostname) {
+            Optional<String> parentHostname,
+            Optional<URI> archiveUri) {
         if (state == NodeState.active) {
             requireOptional(owner, "owner");
             requireOptional(membership, "membership");
@@ -120,6 +123,7 @@ public class NodeSpec {
         this.additionalIpAddresses = Objects.requireNonNull(additionalIpAddresses);
         this.reports = Objects.requireNonNull(reports);
         this.parentHostname = Objects.requireNonNull(parentHostname);
+        this.archiveUri = Objects.requireNonNull(archiveUri);
     }
 
     public String hostname() {
@@ -244,6 +248,10 @@ public class NodeSpec {
         return parentHostname;
     }
 
+    public Optional<URI> archiveUri() {
+        return archiveUri;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -274,7 +282,8 @@ public class NodeSpec {
                 Objects.equals(ipAddresses, that.ipAddresses) &&
                 Objects.equals(additionalIpAddresses, that.additionalIpAddresses) &&
                 Objects.equals(reports, that.reports) &&
-                Objects.equals(parentHostname, that.parentHostname);
+                Objects.equals(parentHostname, that.parentHostname) &&
+                Objects.equals(archiveUri, that.archiveUri);
     }
 
     @Override
@@ -303,7 +312,8 @@ public class NodeSpec {
                 ipAddresses,
                 additionalIpAddresses,
                 reports,
-                parentHostname);
+                parentHostname,
+                archiveUri);
     }
 
     @Override
@@ -333,6 +343,7 @@ public class NodeSpec {
                 + " additionalIpAddresses=" + additionalIpAddresses
                 + " reports=" + reports
                 + " parentHostname=" + parentHostname
+                + " archiveUri=" + archiveUri
                 + " }";
     }
 
@@ -362,6 +373,7 @@ public class NodeSpec {
         private Set<String> additionalIpAddresses = Set.of();
         private NodeReports reports = new NodeReports();
         private Optional<String> parentHostname = Optional.empty();
+        private Optional<URI> archiveUri = Optional.empty();
 
         public Builder() {}
 
@@ -390,6 +402,7 @@ public class NodeSpec {
             node.wantedFirmwareCheck.ifPresent(this::wantedFirmwareCheck);
             node.currentFirmwareCheck.ifPresent(this::currentFirmwareCheck);
             node.parentHostname.ifPresent(this::parentHostname);
+            node.archiveUri.ifPresent(this::archiveUri);
         }
 
         public Builder hostname(String hostname) {
@@ -542,6 +555,11 @@ public class NodeSpec {
             return this;
         }
 
+        public Builder archiveUri(URI archiveUri) {
+            this.archiveUri = Optional.of(archiveUri);
+            return this;
+        }
+
         public Builder updateFromNodeAttributes(NodeAttributes attributes) {
             attributes.getDockerImage().ifPresent(this::currentDockerImage);
             attributes.getCurrentOsVersion().ifPresent(this::currentOsVersion);
@@ -640,6 +658,10 @@ public class NodeSpec {
             return parentHostname;
         }
 
+        public Optional<URI> archiveUri() {
+            return archiveUri;
+        }
+
         public NodeSpec build() {
             return new NodeSpec(hostname, wantedDockerImage, currentDockerImage, state, type, flavor,
                     wantedVespaVersion, currentVespaVersion, wantedOsVersion, currentOsVersion, orchestratorStatus,
@@ -648,7 +670,7 @@ public class NodeSpec {
                     wantedRebootGeneration, currentRebootGeneration,
                     wantedFirmwareCheck, currentFirmwareCheck, modelName,
                     resources, ipAddresses, additionalIpAddresses,
-                    reports, parentHostname);
+                    reports, parentHostname, archiveUri);
         }
 
 
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
index 9ad4d3c565d..c0fd77a89b7 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java
@@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.Ge
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeMessageResponse;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeRepositoryNode;
 
+import java.net.URI;
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
@@ -181,7 +182,8 @@ public class RealNodeRepository implements NodeRepository {
                 node.ipAddresses,
                 node.additionalIpAddresses,
                 reports,
-                Optional.ofNullable(node.parentHostname));
+                Optional.ofNullable(node.parentHostname),
+                Optional.ofNullable(node.archiveUri).map(URI::create));
     }
 
     private static NodeResources.DiskSpeed diskSpeedFromString(String diskSpeed) {
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java
index ff9c502a60f..aeafcbb8088 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/NodeRepositoryNode.java
@@ -76,6 +76,8 @@ public class NodeRepositoryNode {
     public Boolean wantToDeprovision;
     @JsonProperty("orchestratorStatus")
     public String orchestratorStatus;
+    @JsonProperty("archiveUri")
+    public String archiveUri;
 
     @JsonProperty("reports")
     public Map<String, JsonNode> reports = null;
@@ -113,6 +115,7 @@ public class NodeRepositoryNode {
                 ", wantToRetire=" + wantToRetire +
                 ", wantToDeprovision=" + wantToDeprovision +
                 ", orchestratorStatus=" + orchestratorStatus +
+                ", archiveUri=" + archiveUri +
                 ", reports=" + reports +
                 '}';
     }
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
index c30add928ba..6049cc31b68 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java
@@ -3,13 +3,8 @@ package com.yahoo.vespa.hosted.node.admin.maintenance;
 
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
-import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.DockerImage;
 import com.yahoo.config.provision.NodeType;
-import com.yahoo.vespa.flags.FetchVector;
-import com.yahoo.vespa.flags.FlagSource;
-import com.yahoo.vespa.flags.Flags;
-import com.yahoo.vespa.flags.StringFlag;
 import com.yahoo.vespa.hosted.dockerapi.Container;
 import com.yahoo.vespa.hosted.dockerapi.ContainerName;
 import com.yahoo.vespa.hosted.node.admin.component.TaskContext;
@@ -28,6 +23,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder;
 import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath;
 import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal;
 
+import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -66,7 +62,6 @@ public class StorageMaintainer {
     private final SyncClient syncClient;
     private final Clock clock;
     private final Path archiveContainerStoragePath;
-    private final StringFlag syncBucketNameFlag;
 
     // We cache disk usage to avoid doing expensive disk operations so often
     private final Cache<ContainerName, DiskSize> diskUsage = CacheBuilder.newBuilder()
@@ -75,30 +70,24 @@ public class StorageMaintainer {
             .build();
 
     public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, DiskCleanup diskCleanup,
-                             SyncClient syncClient, Clock clock, Path archiveContainerStoragePath, FlagSource flagSource) {
+                             SyncClient syncClient, Clock clock, Path archiveContainerStoragePath) {
         this.terminal = terminal;
         this.coredumpHandler = coredumpHandler;
         this.diskCleanup = diskCleanup;
         this.syncClient = syncClient;
         this.clock = clock;
         this.archiveContainerStoragePath = archiveContainerStoragePath;
-        this.syncBucketNameFlag = Flags.SYNC_HOST_LOGS_TO_S3_BUCKET.bindTo(flagSource);
     }
 
     public boolean syncLogs(NodeAgentContext context) {
-        Optional<ApplicationId> app = context.node().owner();
-        if (app.isEmpty()) return false;
-        String bucketName = syncBucketNameFlag
-                .with(FetchVector.Dimension.NODE_TYPE, NodeType.tenant.name())
-                .with(FetchVector.Dimension.APPLICATION_ID, app.get().serializedForm())
-                .value();
-        if (bucketName.isBlank()) return false;
+        Optional<URI> archiveUri = context.node().archiveUri();
+        if (archiveUri.isEmpty()) return false;
 
         List<SyncFileInfo> syncFileInfos = FileFinder.files(pathOnHostUnderContainerVespaHome(context, "logs/vespa"))
                 .maxDepth(2)
                 .stream()
                 .sorted(Comparator.comparing(FileFinder.FileAttributes::lastModifiedTime))
-                .flatMap(fa -> SyncFileInfo.tenantLog(bucketName, app.get(), context.hostname(), fa.path()).stream())
+                .flatMap(fa -> SyncFileInfo.forLogFile(archiveUri.get(), fa.path()).stream())
                 .collect(Collectors.toList());
 
         return syncClient.sync(context, syncFileInfos, 1);
diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
index 19ec8c0e283..55ead4ad737 100644
--- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
+++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java
@@ -1,11 +1,8 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
 
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
-
+import java.net.URI;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.Optional;
 
 /**
@@ -13,30 +10,24 @@ import java.util.Optional;
  */
 public class SyncFileInfo {
 
-    private final String bucketName;
-    private final Path srcPath;
-    private final Path destPath;
+    private final Path source;
+    private final URI destination;
     private final Compression uploadCompression;
 
-    private SyncFileInfo(String bucketName, Path srcPath, Path destPath, Compression uploadCompression) {
-        this.bucketName = bucketName;
-        this.srcPath = srcPath;
-        this.destPath = destPath;
+    private SyncFileInfo(Path source, URI destination, Compression uploadCompression) {
+        this.source = source;
+        this.destination = destination;
         this.uploadCompression = uploadCompression;
     }
 
-    public String bucketName() {
-        return bucketName;
-    }
-
     /** Source path of the file to sync */
-    public Path srcPath() {
-        return srcPath;
+    public Path source() {
+        return source;
     }
 
-    /** Remote path to store the file at */
-    public Path destPath() {
-        return destPath;
+    /** Remote URI to store the file at */
+    public URI destination() {
+        return destination;
     }
 
     /** Compression algorithm to use when uploading the file */
@@ -44,52 +35,28 @@ public class SyncFileInfo {
         return uploadCompression;
     }
 
-    public static Optional<SyncFileInfo> tenantLog(String bucketName, ApplicationId applicationId, HostName hostName, Path logFile) {
+    public static Optional<SyncFileInfo> forLogFile(URI uri, Path logFile) {
         String filename = logFile.getFileName().toString();
         Compression compression = Compression.NONE;
         String dir = null;
 
         if (filename.startsWith("vespa.log-")) {
-            dir = "logs/vespa";
+            dir = "logs/vespa/";
             compression = Compression.ZSTD;
         } else if (filename.endsWith(".zst")) {
             if (filename.startsWith("JsonAccessLog.") || filename.startsWith("access"))
-                dir = "logs/access";
+                dir = "logs/access/";
             else if (filename.startsWith("ConnectionLog."))
-                dir = "logs/connection";
+                dir = "logs/connection/";
         }
 
         if (dir == null) return Optional.empty();
         return Optional.of(new SyncFileInfo(
-                bucketName, logFile, destination(applicationId, hostName, dir, logFile, compression), compression));
-    }
-
-    public static SyncFileInfo infrastructureVespaLog(String bucketName, HostName hostName, Path vespaLogFile) {
-        Compression compression = Compression.ZSTD;
-        return new SyncFileInfo(bucketName, vespaLogFile, destination(null, hostName, "logs/vespa", vespaLogFile, compression), compression);
-    }
-
-    private static Path destination(ApplicationId app, HostName hostName, String dir, Path filename, Compression uploadCompression) {
-        StringBuilder sb = new StringBuilder(100);
-
-        if (app == null) sb.append("infrastructure");
-        else sb.append(app.tenant().value()).append('/').append(app.application().value()).append('/').append(app.instance().value());
-
-        sb.append('/');
-        for (char c: hostName.value().toCharArray()) {
-            if (c == '.') break;
-            sb.append(c);
-        }
-
-        sb.append('/').append(dir).append('/').append(filename.getFileName().toString());
-
-        if (uploadCompression.extension != null) sb.append(uploadCompression.extension);
-
-        return Paths.get(sb.toString());
+                logFile, uri.resolve(dir + logFile.getFileName() + compression.extension), compression));
     }
 
     public enum Compression {
-        NONE(null), ZSTD(".zst");
+        NONE(""), ZSTD(".zst");
 
         private final String extension;
         Compression(String extension) {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
index a17ffadcb45..beca554fb2d 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.node.admin.maintenance;
 
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.test.ManualClock;
-import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec;
 import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler;
 import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup;
@@ -46,10 +45,9 @@ public class StorageMaintainerTest {
     private final DiskCleanup diskCleanup = mock(DiskCleanup.class);
     private final SyncClient syncClient = mock(SyncClient.class);
     private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(1234567890));
-    private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
     private final FileSystem fileSystem = TestFileSystem.create();
     private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, coredumpHandler, diskCleanup, syncClient, clock,
-            fileSystem.getPath("/home/docker/container-storage/container-archive"), flagSource);
+            fileSystem.getPath("/home/docker/container-storage/container-archive"));
 
     @Test
     public void testDiskUsed() throws IOException {
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
index e32fc8e5355..57f3e2011bd 100644
--- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
@@ -1,11 +1,10 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
 
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.test.file.TestFileSystem;
 import org.junit.Test;
 
+import java.net.URI;
 import java.nio.file.FileSystem;
 import java.nio.file.Path;
 import java.util.Optional;
@@ -21,9 +20,7 @@ public class SyncFileInfoTest {
 
     private static final FileSystem fileSystem = TestFileSystem.create();
 
-    private static final String bucket = "logs-region-acdf21";
-    private static final ApplicationId application = ApplicationId.from("ten", "app", "ins");
-    private static final HostName hostname = HostName.from("h12352a.env.region-1.vespa.domain.example");
+    private static final URI nodeArchiveUri = URI.create("s3://vespa-data-bucket/vespa/music/main/h432a/");
     private static final Path accessLogPath1 = fileSystem.getPath("/opt/vespa/logs/qrs/access.log.20210211");
     private static final Path accessLogPath2 = fileSystem.getPath("/opt/vespa/logs/qrs/access.log.20210212.zst");
     private static final Path accessLogPath3 = fileSystem.getPath("/opt/vespa/logs/qrs/access-json.log.20210213.zst");
@@ -34,29 +31,22 @@ public class SyncFileInfoTest {
     private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12");
 
     @Test
-    public void tenant_log() {
-        assertTenantSyncFileInfo(accessLogPath1, null, null);
-        assertTenantSyncFileInfo(accessLogPath2, "ten/app/ins/h12352a/logs/access/access.log.20210212.zst", NONE);
-        assertTenantSyncFileInfo(accessLogPath3, "ten/app/ins/h12352a/logs/access/access-json.log.20210213.zst", NONE);
-        assertTenantSyncFileInfo(accessLogPath4, "ten/app/ins/h12352a/logs/access/JsonAccessLog.default.20210214.zst", NONE);
+    public void log_files_test() {
+        assertForLogFile(accessLogPath1, null, null);
+        assertForLogFile(accessLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access.log.20210212.zst", NONE);
+        assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE);
+        assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.default.20210214.zst", NONE);
 
-        assertTenantSyncFileInfo(connectionLogPath1, null, null);
-        assertTenantSyncFileInfo(connectionLogPath2, "ten/app/ins/h12352a/logs/connection/ConnectionLog.default.20210212.zst", NONE);
+        assertForLogFile(connectionLogPath1, null, null);
+        assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.default.20210212.zst", NONE);
 
-        assertTenantSyncFileInfo(vespaLogPath1, null, null);
-        assertTenantSyncFileInfo(vespaLogPath2, "ten/app/ins/h12352a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD);
+        assertForLogFile(vespaLogPath1, null, null);
+        assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD);
     }
 
-    @Test
-    public void infra_vespa_log() {
-        SyncFileInfo sfi = SyncFileInfo.infrastructureVespaLog(bucket, hostname, vespaLogPath2);
-        assertEquals("infrastructure/h12352a/logs/vespa/vespa.log-2021-02-12.zst", sfi.destPath().toString());
-        assertEquals(ZSTD, sfi.uploadCompression());
-    }
-
-    private static void assertTenantSyncFileInfo(Path srcPath, String destPath, SyncFileInfo.Compression compression) {
-        Optional<SyncFileInfo> sfi = SyncFileInfo.tenantLog(bucket, application, hostname, srcPath);
-        assertEquals(destPath, sfi.map(SyncFileInfo::destPath).map(Path::toString).orElse(null));
+    private static void assertForLogFile(Path srcPath, String destination, SyncFileInfo.Compression compression) {
+        Optional<SyncFileInfo> sfi = SyncFileInfo.forLogFile(nodeArchiveUri, srcPath);
+        assertEquals(destination, sfi.map(SyncFileInfo::destination).map(URI::toString).orElse(null));
         assertEquals(compression, sfi.map(SyncFileInfo::uploadCompression).orElse(null));
     }
 }