From ffc98bcd55d44f96a896e38507f7f7174989b758 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 14 Jun 2022 12:00:26 +0200 Subject: Revert "hack in zstd compression" --- .../handlers/archive/ArchiverHandler.java | 8 +- .../logserver/handlers/archive/ArchiverPlugin.java | 5 +- .../logserver/handlers/archive/FilesArchived.java | 109 +-------------------- .../handlers/archive/ArchiverHandlerTestCase.java | 11 +-- .../handlers/archive/FilesArchivedTestCase.java | 38 +++---- 5 files changed, 31 insertions(+), 140 deletions(-) diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index 50df160d01f..0b44e47f183 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -86,9 +86,9 @@ public class ArchiverHandler extends AbstractLogHandler { * Creates an ArchiverHandler which puts files under * the given root directory. */ - public ArchiverHandler(String rootDir, int maxFileSize, String zip) { + public ArchiverHandler(String rootDir, int maxFileSize) { this(); - setRootDir(rootDir, zip); + setRootDir(rootDir); this.maxFileSize = maxFileSize; } @@ -189,7 +189,7 @@ public class ArchiverHandler extends AbstractLogHandler { } } - private void setRootDir(String rootDir, String zip) { + private void setRootDir(String rootDir) { // roundabout way of setting things, but this way we can // get around Java's ineptitude for file handling (relative paths in File are broken) absoluteRootDir = new File(rootDir).getAbsolutePath(); @@ -205,7 +205,7 @@ public class ArchiverHandler extends AbstractLogHandler { log.log(Level.FINE, () -> "Created root at " + absoluteRootDir); } } - filesArchived = new FilesArchived(root, zip); + filesArchived = new FilesArchived(root); } public String toString() { diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java index afbd12ab05f..deb3b1adcf4 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverPlugin.java @@ -20,8 +20,6 @@ public class ArchiverPlugin implements Plugin { */ private static final String DEFAULT_MAXFILESIZE = "20971520"; - private static final String DEFAULT_COMPRESSION = "zstd"; - private final Server server = Server.getInstance(); private static final Logger log = Logger.getLogger(ArchiverPlugin.class.getName()); private ArchiverHandler archiver; @@ -54,10 +52,9 @@ public class ArchiverPlugin implements Plugin { String rootDir = config.get("dir", DEFAULT_DIR); int maxFileSize = config.getInt("maxfilesize", DEFAULT_MAXFILESIZE); String threadName = config.get("thread", getPluginName()); - String zip = config.get("compression", DEFAULT_COMPRESSION); // register log handler and flusher - archiver = new ArchiverHandler(rootDir, maxFileSize, zip); + archiver = new ArchiverHandler(rootDir, maxFileSize); server.registerLogHandler(archiver, threadName); server.registerFlusher(archiver); } diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java index d1e9793ffaf..54e47e15d8e 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/FilesArchived.java @@ -8,26 +8,10 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; -import com.yahoo.compress.ZstdOutputStream; -import com.yahoo.io.NativeIO; -import com.yahoo.log.LogFileDb; -import com.yahoo.protect.Process; -import com.yahoo.yolean.Exceptions; - -import java.io.BufferedOutputStream; -import java.io.FileDescriptor; -import java.io.FileNotFoundException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; - /** * This class holds information about all (log) files contained @@ -44,10 +28,6 @@ public class FilesArchived { */ private final File root; - enum Compression {NONE, GZIP, ZSTD} - private final Compression compression; - private final NativeIO nativeIO = new NativeIO(); - private final Object mutex = new Object(); // known-existing files inside the archive directory @@ -80,9 +60,8 @@ public class FilesArchived { /** * Creates an instance of FilesArchive managing the given directory */ - public FilesArchived(File rootDir, String zip) { + public FilesArchived(File rootDir) { this.root = rootDir; - this.compression = ("zstd".equals(zip)) ? Compression.ZSTD : Compression.GZIP; rescan(); Thread thread = new Thread(this::run); thread.setDaemon(true); @@ -173,24 +152,7 @@ public class FilesArchived { return count > 0; } - private void compress(File oldFile) { - switch (compression) { - case ZSTD: - runCompressionZstd(nativeIO, oldFile); - break; - case GZIP: - compressGzip(oldFile); - break; - case NONE: - runCompressionNone(nativeIO, oldFile); - break; - default: - throw new IllegalArgumentException("Unknown compression " + compression); - } - } - - private void compressGzip(File oldFile) { File gzippedFile = new File(oldFile.getPath() + ".gz"); try (GZIPOutputStream compressor = new GZIPOutputStream(new FileOutputStream(gzippedFile), 0x100000); FileInputStream inputStream = new FileInputStream(oldFile)) @@ -211,32 +173,6 @@ public class FilesArchived { } } - private static void runCompressionZstd(NativeIO nativeIO, File oldFile) { - try { - Path compressedFile = Paths.get(oldFile.toString() + ".zst"); - int bufferSize = 2*1024*1024; - long mtime = oldFile.lastModified(); - try (FileOutputStream fileOut = AtomicFileOutputStream.create(compressedFile); - ZstdOutputStream out = new ZstdOutputStream(fileOut, bufferSize); - FileInputStream in = new FileInputStream(oldFile)) - { - pageFriendlyTransfer(nativeIO, out, fileOut.getFD(), in, bufferSize); - out.flush(); - } - compressedFile.toFile().setLastModified(mtime); - oldFile.delete(); - nativeIO.dropFileFromCache(compressedFile.toFile()); - } catch (IOException e) { - log.log(Level.WARNING, "Failed to compress log file with zstd: " + oldFile, e); - } finally { - nativeIO.dropFileFromCache(oldFile); - } - } - - private static void runCompressionNone(NativeIO nativeIO, File oldFile) { - nativeIO.dropFileFromCache(oldFile); - } - long sumFileSizes() { long sum = 0; for (LogFile lf : knownFiles) { @@ -274,7 +210,7 @@ public class FilesArchived { } static class LogFile { - public final File path; + public final File path; public final String prefix; public final int generation; public final boolean zsuff; @@ -309,7 +245,6 @@ public class FilesArchived { } private static boolean zSuffix(String name) { if (name.endsWith(".gz")) return true; - if (name.endsWith(".zst")) return true; // add other compression suffixes here return false; } @@ -324,44 +259,4 @@ public class FilesArchived { return "FilesArchived.LogFile{name="+path+" prefix="+prefix+" gen="+generation+" z="+zsuff+"}"; } } - - private static class AtomicFileOutputStream extends FileOutputStream { - private final Path path; - private final Path tmpPath; - private volatile boolean closed = false; - - private AtomicFileOutputStream(Path path, Path tmpPath) throws FileNotFoundException { - super(tmpPath.toFile()); - this.path = path; - this.tmpPath = tmpPath; - } - - @Override - public synchronized void close() throws IOException { - super.close(); - if (!closed) { - Files.move(tmpPath, path, StandardCopyOption.ATOMIC_MOVE); - closed = true; - } - } - - private static AtomicFileOutputStream create(Path path) throws FileNotFoundException { - return new AtomicFileOutputStream(path, path.resolveSibling("." + path.getFileName() + ".tmp")); - } - } - - private static void pageFriendlyTransfer(NativeIO nativeIO, OutputStream out, FileDescriptor outDescriptor, FileInputStream in, int bufferSize) throws IOException { - int read; - long totalBytesRead = 0; - byte[] buffer = new byte[bufferSize]; - while ((read = in.read(buffer)) >= 0) { - out.write(buffer, 0, read); - if (read > 0) { - nativeIO.dropPartialFileFromCache(in.getFD(), totalBytesRead, read, false); - nativeIO.dropPartialFileFromCache(outDescriptor, totalBytesRead, read, false); - } - totalBytesRead += read; - } - } - } diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java index bffbde67fa4..f3153f7bc14 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/ArchiverHandlerTestCase.java @@ -64,7 +64,7 @@ public class ArchiverHandlerTestCase { File tmpDir = temporaryFolder.newFolder(); ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - 1024, "gzip"); + 1024); long now = 1095159244095L; long midnight = 1095206400000L; assertEquals(2004091410, a.dateHash(now)); @@ -82,7 +82,7 @@ public class ArchiverHandlerTestCase { File tmpDir = temporaryFolder.newFolder(); try { ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - 1024, "gzip"); + 1024); LogMessage msg1 = LogMessage.parseNativeFormat("1139322725\thost\t1/1\tservice\tcomponent\tinfo\tpayload"); LogMessage msg2 = LogMessage.parseNativeFormat("1161172200\thost\t1/1\tservice\tcomponent\tinfo\tpayload"); assertEquals(tmpDir.getAbsolutePath() + "/2006/02/07/14", a.getPrefix(msg1)); @@ -103,7 +103,7 @@ public class ArchiverHandlerTestCase { File tmpDir = temporaryFolder.newFolder(); ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - 1024, "gzip"); + 1024); for (int i = 0; i < msg.length; i++) { a.handle(msg[i]); @@ -168,8 +168,7 @@ public class ArchiverHandlerTestCase { File tmpDir = temporaryFolder.newFolder(); ArchiverHandler a = new ArchiverHandler(tmpDir.getAbsolutePath(), - msg[1].toString().length() + 1, - "gzip"); + msg[1].toString().length() + 1); // log the same message 4 times for (int i = 0; i < 4; i++) { a.handle(msg[1]); @@ -206,7 +205,7 @@ public class ArchiverHandlerTestCase { public void testCacheEldestEntry() throws IOException { LogWriterLRUCache cache = new LogWriterLRUCache(5, (float) 0.75); String d = "target/tmp/logarchive"; - FilesArchived archive = new FilesArchived(new File(d), "gzip"); + FilesArchived archive = new FilesArchived(new File(d)); for (int i = 0; i < cache.maxEntries + 10; i++) { cache.put(i, new LogWriter(d+"/2018/12/31/17", 5, archive)); } diff --git a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java index babe4b1479d..6004df88cfe 100644 --- a/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java +++ b/logserver/src/test/java/com/yahoo/logserver/handlers/archive/FilesArchivedTestCase.java @@ -55,7 +55,7 @@ public class FilesArchivedTestCase { makeLogfile("2018/12/31/16-0", 1); makeLogfile("2018/12/31/17-0", 0); dumpFiles("before archive maintenance"); - FilesArchived a = new FilesArchived(tmpDir, "zstd"); + FilesArchived a = new FilesArchived(tmpDir); dumpFiles("also before archive maintenance"); checkExist("foo/bar"); @@ -67,14 +67,14 @@ public class FilesArchivedTestCase { checkExist("2018/12/31/14-0"); checkExist("2018/12/31/16-0"); checkExist("2018/12/31/17-0"); - checkNoExist("2018/11/20/13-0.zst"); - checkNoExist("2018/11/21/13-0.zst"); - checkNoExist("2018/12/28/13-0.zst"); - checkNoExist("2018/12/29/13-0.zst"); - checkNoExist("2018/12/30/13-0.zst"); - checkNoExist("2018/12/31/14-0.zst"); - checkNoExist("2018/12/31/16-0.zst"); - checkNoExist("2018/12/31/17-0.zst"); + checkNoExist("2018/11/20/13-0.gz"); + checkNoExist("2018/11/21/13-0.gz"); + checkNoExist("2018/12/28/13-0.gz"); + checkNoExist("2018/12/29/13-0.gz"); + checkNoExist("2018/12/30/13-0.gz"); + checkNoExist("2018/12/31/14-0.gz"); + checkNoExist("2018/12/31/16-0.gz"); + checkNoExist("2018/12/31/17-0.gz"); a.maintenance(); @@ -82,22 +82,22 @@ public class FilesArchivedTestCase { checkExist("foo/bar"); checkExist("2018/12/31/17-0"); checkExist("2018/12/31/16-0"); - checkExist("2018/12/31/14-0.zst"); - checkExist("2018/12/28/13-0.zst"); - checkExist("2018/12/29/13-0.zst"); - checkExist("2018/12/30/13-0.zst"); + checkExist("2018/12/31/14-0.gz"); + checkExist("2018/12/28/13-0.gz"); + checkExist("2018/12/29/13-0.gz"); + checkExist("2018/12/30/13-0.gz"); - checkNoExist("2018/12/31/17-0.zst"); - checkNoExist("2018/12/31/16-0.zst"); + checkNoExist("2018/12/31/17-0.gz"); + checkNoExist("2018/12/31/16-0.gz"); checkNoExist("2018/12/31/14-0"); checkNoExist("2018/12/28/13-0"); checkNoExist("2018/12/29/13-0"); checkNoExist("2018/12/30/13-0"); checkNoExist("2018/11/20/13-0"); - checkNoExist("2018/11/20/13-0.zst"); + checkNoExist("2018/11/20/13-0.gz"); checkNoExist("2018/11/21/13-0"); - checkNoExist("2018/11/21/13-0.zst"); + checkNoExist("2018/11/21/13-0.gz"); makeLogfile("2018/12/31/16-0", 3); makeLogfile("2018/12/31/17-0", 3); @@ -110,8 +110,8 @@ public class FilesArchivedTestCase { checkExist("2018/12/31/17-2"); checkExist("2018/12/31/17-1"); - checkExist("2018/12/31/16-0.zst"); - checkExist("2018/12/31/17-0.zst"); + checkExist("2018/12/31/16-0.gz"); + checkExist("2018/12/31/17-0.gz"); checkNoExist("2018/12/31/16-0"); checkNoExist("2018/12/31/17-0"); -- cgit v1.2.3