Define SyncClient

2 files changed, 124 insertions, 0 deletions

diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
new file mode 100644
index 00000000000..0d596a46d77
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java
@@ -0,0 +1,79 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.test.file.TestFileSystem;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+/**
+ * @author freva
+ */
+public class SyncFileInfoTest {
+
+    private static final FileSystem fileSystem = TestFileSystem.create();
+
+    private static final String bucket = "logs-region-acdf21";
+    private static final ApplicationId application = ApplicationId.from("tenant", "application", "instance");
+    private static final HostName hostname = HostName.from("h12352a.env.region-1.vespa.domain.example");
+    private static final Path accessLogPath = fileSystem.getPath("/opt/vespa/logs/qrs/access.json-20210212.zst");
+    private static final Path vespaLogPath = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12");
+
+    @Test
+    public void tenant_access_log() {
+        SyncFileInfo sfi = SyncFileInfo.tenantAccessLog(bucket, application, hostname, accessLogPath);
+        assertEquals(Paths.get("/tenant.application.instance/h12352a/logs/access/access.json-20210212.zst"), sfi.destPath());
+        assertEquals(bucket, sfi.bucketName());
+        assertNotEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi));
+    }
+
+    @Test
+    public void tenant_vespa_log() {
+        SyncFileInfo sfi = SyncFileInfo.tenantVespaLog(bucket, application, hostname, vespaLogPath);
+        assertEquals(Paths.get("/tenant.application.instance/h12352a/logs/vespa/vespa.log-2021-02-12.zst"), sfi.destPath());
+        assertEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi));
+    }
+
+    @Test
+    public void infra_access_log() {
+        SyncFileInfo sfi = SyncFileInfo.infrastructureAccessLog(bucket, hostname, accessLogPath);
+        assertEquals(Paths.get("/infrastructure/h12352a/logs/access/access.json-20210212.zst"), sfi.destPath());
+        assertNotEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi));
+    }
+
+    @Test
+    public void infra_vespa_log() {
+        SyncFileInfo sfi = SyncFileInfo.infrastructureVespaLog(bucket, hostname, vespaLogPath);
+        assertEquals(Paths.get("/infrastructure/h12352a/logs/vespa/vespa.log-2021-02-12.zst"), sfi.destPath());
+        assertEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi));
+    }
+
+    @BeforeClass
+    public static void setup() throws IOException {
+        Files.createDirectories(vespaLogPath.getParent());
+        Files.createFile(vespaLogPath);
+        Files.createDirectories(accessLogPath.getParent());
+        Files.createFile(accessLogPath);
+    }
+
+    private static Class<? extends InputStream> getInputStreamType(SyncFileInfo syncFileInfo) {
+        try (InputStream inputStream = syncFileInfo.inputStream()) {
+            return inputStream.getClass();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java
new file mode 100644
index 00000000000..be0df437b7f
--- /dev/null
+++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java
@@ -0,0 +1,45 @@
+package com.yahoo.vespa.hosted.node.admin.maintenance.sync;
+
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Random;
+
+import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream.compressor;
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * @author freva
+ */
+public class ZstdCompressingInputStreamTest {
+
+    @Test
+    public void compression_test() throws Exception {
+        Random rnd = new Random();
+        byte[] data = new byte[(int) (100_000 * (10 + rnd.nextDouble()))];
+        rnd.nextBytes(data);
+        assertCompression(data, 1 << 14);
+    }
+
+    private static void assertCompression(byte[] data, int bufferSize) {
+        ByteArrayInputStream bais = new ByteArrayInputStream(data);
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(bais, bufferSize)) {
+            byte[] buffer = new byte[bufferSize];
+            for (int nRead; (nRead = zcis.read(buffer, 0, buffer.length)) != -1; )
+                baos.write(buffer, 0, nRead);
+            baos.flush();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
+        byte[] compressedData = baos.toByteArray();
+        byte[] decompressedData = new byte[data.length];
+        compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length);
+
+        assertArrayEquals(data, decompressedData);
+    }
+}