From 6930d2112dd39b8362df0c9ade7800f3bd3bebda Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 16 Apr 2021 23:33:00 +0200 Subject: - Use 256k buffer for connection log as it is low volume. - Used 256k buffer accesslog, but 4m for application containers. --- .../yahoo/container/logging/AccessLogHandler.java | 2 +- .../container/logging/ConnectionLogHandler.java | 4 +++- .../yahoo/container/logging/FileConnectionLog.java | 2 +- .../yahoo/container/logging/LogFileHandler.java | 23 ++++++++++------------ .../container.core.access-log.def | 3 +++ .../container.logging.connection-log.def | 5 ++++- .../container/logging/LogFileHandlerTestCase.java | 15 +++++++------- 7 files changed, 30 insertions(+), 24 deletions(-) (limited to 'container-core') diff --git a/container-core/src/main/java/com/yahoo/container/logging/AccessLogHandler.java b/container-core/src/main/java/com/yahoo/container/logging/AccessLogHandler.java index 89aab1513ee..f14479899f5 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/AccessLogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/logging/AccessLogHandler.java @@ -12,7 +12,7 @@ class AccessLogHandler { AccessLogHandler(AccessLogConfig.FileHandler config, LogWriter logWriter) { logFileHandler = new LogFileHandler<>( - toCompression(config), config.pattern(), config.rotation(), + toCompression(config), config.bufferSize(), config.pattern(), config.rotation(), config.symlink(), config.queueSize(), "request-logger", logWriter); } diff --git a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java index 7a0e8aca95e..7b130884667 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java @@ -8,9 +8,11 @@ package com.yahoo.container.logging; class ConnectionLogHandler { private final LogFileHandler logFileHandler; - public ConnectionLogHandler(String logDirectoryName, String clusterName, int queueSize, LogWriter logWriter) { + public ConnectionLogHandler(String logDirectoryName, int bufferSize, String clusterName, + int queueSize, LogWriter logWriter) { logFileHandler = new LogFileHandler<>( LogFileHandler.Compression.ZSTD, + bufferSize, String.format("logs/vespa/%s/ConnectionLog.%s.%s", logDirectoryName, clusterName, "%Y%m%d%H%M%S"), "0 60 ...", String.format("ConnectionLog.%s", clusterName), diff --git a/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java b/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java index 7432c313286..749426d3da9 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java +++ b/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java @@ -14,7 +14,7 @@ public class FileConnectionLog extends AbstractComponent implements ConnectionLo @Inject public FileConnectionLog(ConnectionLogConfig config) { - logHandler = new ConnectionLogHandler(config.logDirectoryName(), config.cluster(), config.queueSize(), new JsonConnectionLogWriter()); + logHandler = new ConnectionLogHandler(config.logDirectoryName(), config.bufferSize(), config.cluster(), config.queueSize(), new JsonConnectionLogWriter()); } @Override diff --git a/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java index 0f2a9e42eb8..85c211c0e3a 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-core/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -47,21 +47,15 @@ class LogFileHandler { @FunctionalInterface private interface Pollable { Operation poll() throws InterruptedException; } - LogFileHandler(Compression compression, String filePattern, String rotationTimes, String symlinkName, int queueSize, - String threadName, LogWriter logWriter) { - this(compression, filePattern, calcTimesMinutes(rotationTimes), symlinkName, queueSize, threadName, logWriter); + LogFileHandler(Compression compression, int bufferSize, String filePattern, String rotationTimes, String symlinkName, + int queueSize, String threadName, LogWriter logWriter) { + this(compression, bufferSize, filePattern, calcTimesMinutes(rotationTimes), symlinkName, queueSize, threadName, logWriter); } - LogFileHandler( - Compression compression, - String filePattern, - long[] rotationTimes, - String symlinkName, - int queueSize, - String threadName, - LogWriter logWriter) { + LogFileHandler(Compression compression, int bufferSize, String filePattern, long[] rotationTimes, String symlinkName, + int queueSize, String threadName, LogWriter logWriter) { this.logQueue = new LinkedBlockingQueue<>(queueSize); - this.logThread = new LogThread<>(logWriter, filePattern, compression, rotationTimes, symlinkName, threadName, this::poll); + this.logThread = new LogThread<>(logWriter, filePattern, compression, bufferSize, rotationTimes, symlinkName, threadName, this::poll); this.logThread.start(); } @@ -197,6 +191,7 @@ class LogFileHandler { private volatile String fileName; private final LogWriter logWriter; private final Compression compression; + private final int bufferSize; private final long[] rotationTimes; private final String symlinkName; private final ExecutorService executor = createCompressionTaskExecutor(); @@ -206,6 +201,7 @@ class LogFileHandler { LogThread(LogWriter logWriter, String filePattern, Compression compression, + int bufferSize, long[] rotationTimes, String symlinkName, String threadName, @@ -215,6 +211,7 @@ class LogFileHandler { this.logWriter = logWriter; this.filePattern = filePattern; this.compression = compression; + this.bufferSize = bufferSize; this.rotationTimes = rotationTimes; this.symlinkName = (symlinkName != null && !symlinkName.isBlank()) ? symlinkName : null; this.operationProvider = operationProvider; @@ -360,7 +357,7 @@ class LogFileHandler { internalClose(); try { checkAndCreateDir(fileName); - fileOutput = new PageCacheFriendlyFileOutputStream(nativeIO, Paths.get(fileName), 4 * 1024 * 1024); + fileOutput = new PageCacheFriendlyFileOutputStream(nativeIO, Paths.get(fileName), bufferSize); LogFileDb.nowLoggingTo(fileName); } catch (IOException e) { throw new RuntimeException("Couldn't open log file '" + fileName + "'", e); diff --git a/container-core/src/main/resources/configdefinitions/container.core.access-log.def b/container-core/src/main/resources/configdefinitions/container.core.access-log.def index 69058b3d8da..e6052b7068c 100644 --- a/container-core/src/main/resources/configdefinitions/container.core.access-log.def +++ b/container-core/src/main/resources/configdefinitions/container.core.access-log.def @@ -21,3 +21,6 @@ fileHandler.compressionFormat enum {GZIP, ZSTD} default=GZIP # Max queue length of file handler fileHandler.queueSize int default=10000 + +# Buffer size for the output stream has a default of 256k +fileHandler.bufferSize int default=262144 diff --git a/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def b/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def index 65b632c9008..cb2145cd01c 100644 --- a/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def +++ b/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def @@ -8,4 +8,7 @@ cluster string logDirectoryName string default="qrs" # Max queue length of file handler -queueSize int default=10000 \ No newline at end of file +queueSize int default=10000 + +# Buffer size for the output stream has a default of 256k +bufferSize int default=262144 diff --git a/container-core/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java b/container-core/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java index dad8f5e3f90..d43e3dcebbe 100644 --- a/container-core/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/logging/LogFileHandlerTestCase.java @@ -33,6 +33,7 @@ import static org.junit.Assert.assertNotEquals; * @author bjorncs */ public class LogFileHandlerTestCase { + private static final int BUFFER_SIZE = 0x10000; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -43,7 +44,7 @@ public class LogFileHandlerTestCase { String pattern = root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S"; long[] rTimes = {1000, 2000, 10000}; - LogFileHandler h = new LogFileHandler<>(Compression.NONE, pattern, rTimes, null, 2048, "thread-name", new StringLogWriter()); + LogFileHandler h = new LogFileHandler<>(Compression.NONE, BUFFER_SIZE, pattern, rTimes, null, 2048, "thread-name", new StringLogWriter()); long now = System.currentTimeMillis(); long millisPerDay = 60*60*24*1000; long tomorrowDays = (now / millisPerDay) +1; @@ -65,7 +66,7 @@ public class LogFileHandlerTestCase { File logFile = temporaryFolder.newFile("testLogFileG1.txt"); //create logfilehandler - LogFileHandler h = new LogFileHandler<>(Compression.NONE, logFile.getAbsolutePath(), "0 5 ...", null, 2048, "thread-name", new StringLogWriter()); + LogFileHandler h = new LogFileHandler<>(Compression.NONE, BUFFER_SIZE, logFile.getAbsolutePath(), "0 5 ...", null, 2048, "thread-name", new StringLogWriter()); //write log h.publish("testDeleteFileFirst1"); @@ -78,7 +79,7 @@ public class LogFileHandlerTestCase { File logFile = temporaryFolder.newFile("testLogFileG2.txt"); //create logfilehandler - LogFileHandler h = new LogFileHandler<>(Compression.NONE, logFile.getAbsolutePath(), "0 5 ...", null, 2048, "thread-name", new StringLogWriter()); + LogFileHandler h = new LogFileHandler<>(Compression.NONE, BUFFER_SIZE, logFile.getAbsolutePath(), "0 5 ...", null, 2048, "thread-name", new StringLogWriter()); //write log h.publish("testDeleteFileDuringLogging1"); @@ -104,7 +105,7 @@ public class LogFileHandlerTestCase { } }; LogFileHandler handler = new LogFileHandler<>( - Compression.NONE, root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s", new long[]{0}, "symlink", 2048, "thread-name", new StringLogWriter()); + Compression.NONE, BUFFER_SIZE, root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s", new long[]{0}, "symlink", 2048, "thread-name", new StringLogWriter()); String message = formatter.format(new LogRecord(Level.INFO, "test")); handler.publishAndWait(message); @@ -128,7 +129,7 @@ public class LogFileHandlerTestCase { public void compresses_previous_log_file() throws InterruptedException, IOException { File root = temporaryFolder.newFolder("compressespreviouslogfile"); LogFileHandler firstHandler = new LogFileHandler<>( - Compression.ZSTD, root.getAbsolutePath() + "/compressespreviouslogfile.%Y%m%d%H%M%S%s", new long[]{0}, "symlink", 2048, "thread-name", new StringLogWriter()); + Compression.ZSTD, BUFFER_SIZE, root.getAbsolutePath() + "/compressespreviouslogfile.%Y%m%d%H%M%S%s", new long[]{0}, "symlink", 2048, "thread-name", new StringLogWriter()); firstHandler.publishAndWait("test"); firstHandler.shutdown(); @@ -136,7 +137,7 @@ public class LogFileHandlerTestCase { assertThat(root.toPath().resolve("symlink").toRealPath().toString()).isEqualTo(firstHandler.getFileName()); LogFileHandler secondHandler = new LogFileHandler<>( - Compression.ZSTD, root.getAbsolutePath() + "/compressespreviouslogfile.%Y%m%d%H%M%S%s", new long[]{0}, "symlink", 2048, "thread-name", new StringLogWriter()); + Compression.ZSTD, BUFFER_SIZE, root.getAbsolutePath() + "/compressespreviouslogfile.%Y%m%d%H%M%S%s", new long[]{0}, "symlink", 2048, "thread-name", new StringLogWriter()); secondHandler.publishAndWait("test"); secondHandler.rotateNow(); @@ -174,7 +175,7 @@ public class LogFileHandlerTestCase { File root = temporaryFolder.newFolder("testcompression" + compression.name()); LogFileHandler h = new LogFileHandler<>( - compression, root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s", new long[]{0}, null, 2048, "thread-name", new StringLogWriter()); + compression, BUFFER_SIZE, root.getAbsolutePath() + "/logfilehandlertest.%Y%m%d%H%M%S%s", new long[]{0}, null, 2048, "thread-name", new StringLogWriter()); int logEntries = 10000; for (int i = 0; i < logEntries; i++) { h.publish("test"); -- cgit v1.2.3