Revert "hack in zstd compression"

5 files changed, 31 insertions, 140 deletions

diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java
index 50df160d01f..0b44e47f183 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java
@@ -86,9 +86,9 @@ public class ArchiverHandler extends AbstractLogHandler {
      * Creates an ArchiverHandler which puts files under
      * the given root directory.
      */
-    public ArchiverHandler(String rootDir, int maxFileSize, String zip) {
+    public ArchiverHandler(String rootDir, int maxFileSize) {
         this();
-        setRootDir(rootDir, zip);
+        setRootDir(rootDir);
         this.maxFileSize = maxFileSize;
     }
 
@@ -189,7 +189,7 @@ public class ArchiverHandler extends AbstractLogHandler {
         }
     }
 
-    private void setRootDir(String rootDir, String zip) {
+    private void setRootDir(String rootDir) {
         // roundabout way of setting things, but this way we can
         // get around Java's ineptitude for file handling (relative paths in File are broken)
         absoluteRootDir = new File(rootDir).getAbsolutePath();
@@ -205,7 +205,7 @@ public class ArchiverHandler extends AbstractLogHandler {
                 log.log(Level.FINE, () -> "Created root at " + absoluteRootDir);
             }
         }
-        filesArchived = new FilesArchived(root, zip);
+        filesArchived = new FilesArchived(root);
     }
 
     public String toString() {
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java
index afbd12ab05f..deb3b1adcf4 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java
@@ -20,8 +20,6 @@ public class ArchiverPlugin implements Plugin {
      */
     private static final String DEFAULT_MAXFILESIZE = "20971520";
 
-    private static final String DEFAULT_COMPRESSION = "zstd";
-
     private final Server server = Server.getInstance();
     private static final Logger log = Logger.getLogger(ArchiverPlugin.class.getName());
     private ArchiverHandler archiver;
@@ -54,10 +52,9 @@ public class ArchiverPlugin implements Plugin {
         String rootDir = config.get("dir", DEFAULT_DIR);
         int maxFileSize = config.getInt("maxfilesize", DEFAULT_MAXFILESIZE);
         String threadName = config.get("thread", getPluginName());
-        String zip = config.get("compression", DEFAULT_COMPRESSION);
 
         // register log handler and flusher
-        archiver = new ArchiverHandler(rootDir, maxFileSize, zip);
+        archiver = new ArchiverHandler(rootDir, maxFileSize);
         server.registerLogHandler(archiver, threadName);
         server.registerFlusher(archiver);
     }
diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
index d1e9793ffaf..54e47e15d8e 100644
--- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
+++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java
@@ -8,26 +8,10 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPOutputStream;
 
-import com.yahoo.compress.ZstdOutputStream;
-import com.yahoo.io.NativeIO;
-import com.yahoo.log.LogFileDb;
-import com.yahoo.protect.Process;
-import com.yahoo.yolean.Exceptions;
-
-import java.io.BufferedOutputStream;
-import java.io.FileDescriptor;
-import java.io.FileNotFoundException;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-
 
 /**
  * This class holds information about all (log) files contained
@@ -44,10 +28,6 @@ public class FilesArchived {
      */
     private final File root;
 
-    enum Compression {NONE, GZIP, ZSTD}
-    private final Compression compression;
-    private final NativeIO nativeIO = new NativeIO();
-
     private final Object mutex = new Object();
 
     // known-existing files inside the archive directory
@@ -80,9 +60,8 @@ public class FilesArchived {
     /**
      * Creates an instance of FilesArchive managing the given directory
      */
-    public FilesArchived(File rootDir, String zip) {
+    public FilesArchived(File rootDir) {
         this.root = rootDir;
-        this.compression = ("zstd".equals(zip)) ? Compression.ZSTD : Compression.GZIP;
         rescan();
         Thread thread = new Thread(this::run);
         thread.setDaemon(true);
@@ -173,24 +152,7 @@ public class FilesArchived {
         return count > 0;
     }
 
-
     private void compress(File oldFile) {
-        switch (compression) {
-        case ZSTD:
-            runCompressionZstd(nativeIO, oldFile);
-            break;
-        case GZIP:
-            compressGzip(oldFile);
-            break;
-        case NONE:
-            runCompressionNone(nativeIO, oldFile);
-            break;
-        default:
-            throw new IllegalArgumentException("Unknown compression " + compression);
-        }
-    }
-
-    private void compressGzip(File oldFile) {
         File gzippedFile = new File(oldFile.getPath() + ".gz");
         try (GZIPOutputStream compressor = new GZIPOutputStream(new FileOutputStream(gzippedFile), 0x100000);
              FileInputStream inputStream = new FileInputStream(oldFile))
@@ -211,32 +173,6 @@ public class FilesArchived {
         }
     }
 
-    private static void runCompressionZstd(NativeIO nativeIO, File oldFile) {
-        try {
-            Path compressedFile = Paths.get(oldFile.toString() + ".zst");
-            int bufferSize = 2*1024*1024;
-            long mtime = oldFile.lastModified();
-            try (FileOutputStream fileOut = AtomicFileOutputStream.create(compressedFile);
-                 ZstdOutputStream out = new ZstdOutputStream(fileOut, bufferSize);
-                 FileInputStream in = new FileInputStream(oldFile))
-            {
-                pageFriendlyTransfer(nativeIO, out, fileOut.getFD(), in, bufferSize);
-                out.flush();
-            }
-            compressedFile.toFile().setLastModified(mtime);
-            oldFile.delete();
-            nativeIO.dropFileFromCache(compressedFile.toFile());
-        } catch (IOException e) {
-            log.log(Level.WARNING, "Failed to compress log file with zstd: " + oldFile, e);
-        } finally {
-            nativeIO.dropFileFromCache(oldFile);
-        }
-    }
-
-    private static void runCompressionNone(NativeIO nativeIO, File oldFile) {
-        nativeIO.dropFileFromCache(oldFile);
-    }
-
     long sumFileSizes() {
         long sum = 0;
         for (LogFile lf : knownFiles) {
@@ -274,7 +210,7 @@ public class FilesArchived {
     }
 
     static class LogFile {
-        public final File path;
+        public final File path; 
         public final String prefix;
         public final int generation;
         public final boolean zsuff;
@@ -309,7 +245,6 @@ public class FilesArchived {
         }
         private static boolean zSuffix(String name) {
             if (name.endsWith(".gz")) return true;
-            if (name.endsWith(".zst")) return true;
             // add other compression suffixes here
             return false;
         }
@@ -324,44 +259,4 @@ public class FilesArchived {
             return "FilesArchived.LogFile{name="+path+" prefix="+prefix+" gen="+generation+" z="+zsuff+"}";
         }
     }
-
-    private static class AtomicFileOutputStream extends FileOutputStream {
-        private final Path path;
-        private final Path tmpPath;
-        private volatile boolean closed = false;
-
-        private AtomicFileOutputStream(Path path, Path tmpPath) throws FileNotFoundException {
-            super(tmpPath.toFile());
-            this.path = path;
-            this.tmpPath = tmpPath;
-        }
-
-        @Override
-        public synchronized void close() throws IOException {
-            super.close();
-            if (!closed) {
-                Files.move(tmpPath, path, StandardCopyOption.ATOMIC_MOVE);
-                closed = true;
-            }
-        }
-
-        private static AtomicFileOutputStream create(Path path) throws FileNotFoundException {
-            return new AtomicFileOutputStream(path, path.resolveSibling("." + path.getFileName() + ".tmp"));
-        }
-    }
-
-    private static void pageFriendlyTransfer(NativeIO nativeIO, OutputStream out, FileDescriptor outDescriptor, FileInputStream in, int bufferSize) throws IOException {
-        int read;
-        long totalBytesRead = 0;
-        byte[] buffer = new byte[bufferSize];
-        while ((read = in.read(buffer)) >= 0) {
-            out.write(buffer, 0, read);
-            if (read > 0) {
-                nativeIO.dropPartialFileFromCache(in.getFD(), totalBytesRead, read, false);
-                nativeIO.dropPartialFileFromCache(outDescriptor, totalBytesRead, read, false);
-            }
-            totalBytesRead += read;
-        }
-    }
-
 }
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java
index bffbde67fa4..f3153f7bc14 100644
--- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java
@@ -64,7 +64,7 @@ public class ArchiverHandlerTestCase {
         File tmpDir = temporaryFolder.newFolder();
 
         ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(),
-                                                1024, "gzip");
+                                                1024);
         long now = 1095159244095L;
         long midnight = 1095206400000L;
         assertEquals(2004091410, a.dateHash(now));
@@ -82,7 +82,7 @@ public class ArchiverHandlerTestCase {
         File tmpDir = temporaryFolder.newFolder();
         try {
             ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(),
-                                                    1024, "gzip");
+                                                    1024);
             LogMessage msg1 = LogMessage.parseNativeFormat("1139322725\thost\t1/1\tservice\tcomponent\tinfo\tpayload");
             LogMessage msg2 = LogMessage.parseNativeFormat("1161172200\thost\t1/1\tservice\tcomponent\tinfo\tpayload");
             assertEquals(tmpDir.getAbsolutePath() + "/2006/02/07/14", a.getPrefix(msg1));
@@ -103,7 +103,7 @@ public class ArchiverHandlerTestCase {
         File tmpDir = temporaryFolder.newFolder();
 
         ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(),
-                                                1024, "gzip");
+                                                1024);
 
         for (int i = 0; i < msg.length; i++) {
             a.handle(msg[i]);
@@ -168,8 +168,7 @@ public class ArchiverHandlerTestCase {
         File tmpDir = temporaryFolder.newFolder();
 
         ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(),
-                                                msg[1].toString().length() + 1,
-                                                "gzip");
+                                                msg[1].toString().length() + 1);
         // log the same message 4 times
         for (int i = 0; i < 4; i++) {
             a.handle(msg[1]);
@@ -206,7 +205,7 @@ public class ArchiverHandlerTestCase {
     public void testCacheEldestEntry() throws IOException {
         LogWriterLRUCache cache = new LogWriterLRUCache(5, (float) 0.75);
         String d = "target/tmp/logarchive";
-        FilesArchived archive = new FilesArchived(new File(d), "gzip");
+        FilesArchived archive = new FilesArchived(new File(d));
         for (int i = 0; i < cache.maxEntries + 10; i++) {
             cache.put(i, new LogWriter(d+"/2018/12/31/17", 5, archive));
         }
diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java
index babe4b1479d..6004df88cfe 100644
--- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java
+++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java
@@ -55,7 +55,7 @@ public class FilesArchivedTestCase {
         makeLogfile("2018/12/31/16-0", 1);
         makeLogfile("2018/12/31/17-0", 0);
         dumpFiles("before archive maintenance");
-        FilesArchived a = new FilesArchived(tmpDir, "zstd");
+        FilesArchived a = new FilesArchived(tmpDir);
         dumpFiles("also before archive maintenance");
 
         checkExist("foo/bar");
@@ -67,14 +67,14 @@ public class FilesArchivedTestCase {
         checkExist("2018/12/31/14-0");
         checkExist("2018/12/31/16-0");
         checkExist("2018/12/31/17-0");
-        checkNoExist("2018/11/20/13-0.zst");
-        checkNoExist("2018/11/21/13-0.zst");
-        checkNoExist("2018/12/28/13-0.zst");
-        checkNoExist("2018/12/29/13-0.zst");
-        checkNoExist("2018/12/30/13-0.zst");
-        checkNoExist("2018/12/31/14-0.zst");
-        checkNoExist("2018/12/31/16-0.zst");
-        checkNoExist("2018/12/31/17-0.zst");
+        checkNoExist("2018/11/20/13-0.gz");
+        checkNoExist("2018/11/21/13-0.gz");
+        checkNoExist("2018/12/28/13-0.gz");
+        checkNoExist("2018/12/29/13-0.gz");
+        checkNoExist("2018/12/30/13-0.gz");
+        checkNoExist("2018/12/31/14-0.gz");
+        checkNoExist("2018/12/31/16-0.gz");
+        checkNoExist("2018/12/31/17-0.gz");
 
         a.maintenance();
 
@@ -82,22 +82,22 @@ public class FilesArchivedTestCase {
         checkExist("foo/bar");
         checkExist("2018/12/31/17-0");
         checkExist("2018/12/31/16-0");
-        checkExist("2018/12/31/14-0.zst");
-        checkExist("2018/12/28/13-0.zst");
-        checkExist("2018/12/29/13-0.zst");
-        checkExist("2018/12/30/13-0.zst");
+        checkExist("2018/12/31/14-0.gz");
+        checkExist("2018/12/28/13-0.gz");
+        checkExist("2018/12/29/13-0.gz");
+        checkExist("2018/12/30/13-0.gz");
 
-        checkNoExist("2018/12/31/17-0.zst");
-        checkNoExist("2018/12/31/16-0.zst");
+        checkNoExist("2018/12/31/17-0.gz");
+        checkNoExist("2018/12/31/16-0.gz");
         checkNoExist("2018/12/31/14-0");
         checkNoExist("2018/12/28/13-0");
         checkNoExist("2018/12/29/13-0");
         checkNoExist("2018/12/30/13-0");
 
         checkNoExist("2018/11/20/13-0");
-        checkNoExist("2018/11/20/13-0.zst");
+        checkNoExist("2018/11/20/13-0.gz");
         checkNoExist("2018/11/21/13-0");
-        checkNoExist("2018/11/21/13-0.zst");
+        checkNoExist("2018/11/21/13-0.gz");
 
         makeLogfile("2018/12/31/16-0", 3);
         makeLogfile("2018/12/31/17-0", 3);
@@ -110,8 +110,8 @@ public class FilesArchivedTestCase {
 
         checkExist("2018/12/31/17-2");
         checkExist("2018/12/31/17-1");
-        checkExist("2018/12/31/16-0.zst");
-        checkExist("2018/12/31/17-0.zst");
+        checkExist("2018/12/31/16-0.gz");
+        checkExist("2018/12/31/17-0.gz");
 
         checkNoExist("2018/12/31/16-0");
         checkNoExist("2018/12/31/17-0");