diff options
author | Harald Musum <musum@verizonmedia.com> | 2022-10-05 11:10:18 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-10-05 11:10:18 +0200 |
commit | cabdbc5226ba17de97e2c92d2c0fcd9982b96856 (patch) | |
tree | 9ab159ac3a1c6456a5c980991be218e6ab11d7f2 /container-search | |
parent | 8fb24f9d9d5939259e048efff7f3e246e4deaeaa (diff) | |
parent | b1207c7fd3c27cd7229425841f99abb87c101de8 (diff) |
Merge pull request #24181 from vespa-engine/lesters/draft-external-logger
Add external logger draft code
Diffstat (limited to 'container-search')
9 files changed, 450 insertions, 0 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 93e8ce16b6f..df3373ee805 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -4581,6 +4581,121 @@ ], "fields": [] }, + "com.yahoo.search.logging.LocalDiskLogger": { + "superClass": "com.yahoo.search.logging.AbstractThreadedLogger", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.search.logging.LocalDiskLoggerConfig)", + "public bridge synthetic boolean send(com.yahoo.search.logging.LoggerEntry)" + ], + "fields": [] + }, + "com.yahoo.search.logging.LocalDiskLoggerConfig$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.search.logging.LocalDiskLoggerConfig)", + "public com.yahoo.search.logging.LocalDiskLoggerConfig$Builder path(java.lang.String)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public com.yahoo.search.logging.LocalDiskLoggerConfig build()" + ], + "fields": [] + }, + "com.yahoo.search.logging.LocalDiskLoggerConfig$Producer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void getConfig(com.yahoo.search.logging.LocalDiskLoggerConfig$Builder)" + ], + "fields": [] + }, + "com.yahoo.search.logging.LocalDiskLoggerConfig": { + "superClass": "com.yahoo.config.ConfigInstance", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public void <init>(com.yahoo.search.logging.LocalDiskLoggerConfig$Builder)", + "public java.lang.String path()" + ], + "fields": [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, + "com.yahoo.search.logging.Logger": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public com.yahoo.search.logging.LoggerEntry$Builder newEntry()", + "public abstract boolean send(com.yahoo.search.logging.LoggerEntry)" + ], + "fields": [] + }, + "com.yahoo.search.logging.LoggerEntry$Builder": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.search.logging.Logger)", + "public com.yahoo.search.logging.LoggerEntry$Builder timestamp(long)", + "public com.yahoo.search.logging.LoggerEntry$Builder query(com.yahoo.search.Query)", + "public com.yahoo.search.logging.LoggerEntry$Builder blob(byte[])", + "public boolean send()" + ], + "fields": [] + }, + "com.yahoo.search.logging.LoggerEntry": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public java.lang.Long timestamp()", + "public com.yahoo.search.Query query()", + "public java.lang.String queryString()", + "public java.nio.ByteBuffer blob()", + "public java.lang.String toString()", + "public java.lang.String toJson()" + ], + "fields": [] + }, "com.yahoo.search.pagetemplates.PageTemplate": { "superClass": "com.yahoo.component.provider.FreezableComponent", "interfaces": [ diff --git a/container-search/src/main/java/com/yahoo/search/logging/AbstractSpoolingLogger.java b/container-search/src/main/java/com/yahoo/search/logging/AbstractSpoolingLogger.java new file mode 100644 index 00000000000..cfcbf2cab45 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/logging/AbstractSpoolingLogger.java @@ -0,0 +1,12 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.logging; + +abstract class AbstractSpoolingLogger extends AbstractThreadedLogger { + + @Override + protected void dequeue(LoggerEntry entry) { + // Todo: add to spooler etc + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/logging/AbstractThreadedLogger.java b/container-search/src/main/java/com/yahoo/search/logging/AbstractThreadedLogger.java new file mode 100644 index 00000000000..5c14109b26e --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/logging/AbstractThreadedLogger.java @@ -0,0 +1,111 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.logging; + +import org.slf4j.LoggerFactory; + +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +abstract class AbstractThreadedLogger implements Logger { + + private final static org.slf4j.Logger log = LoggerFactory.getLogger(AbstractThreadedLogger.class); + + final static int DEFAULT_MAX_THREADS = 1; + final static int DEFAULT_QUEUE_SIZE = 1000; + + private final WorkerThreadExecutor executor; + + AbstractThreadedLogger() { + this(DEFAULT_MAX_THREADS, DEFAULT_QUEUE_SIZE); + } + + AbstractThreadedLogger(int threads, int queueSize) { + executor = new WorkerThreadExecutor(threads, queueSize); + } + + AbstractThreadedLogger(int threads, int queueSize, ThreadFactory factory) { + executor = new WorkerThreadExecutor(threads, queueSize, factory); + } + + @Override + public boolean send(LoggerEntry entry) { + return enqueue(entry); + } + + protected boolean enqueue(LoggerEntry entry) { + // Todo: metric things + try { + executor.execute(() -> dequeue(entry)); + } catch (RejectedExecutionException e) { + return false; + } + return true; + } + + protected void dequeue(LoggerEntry entry) { + transport(entry); // This happens in worker thread + } + + /** + * Actually transports the entry to it's destination + */ + abstract void transport(LoggerEntry entry); + + + private static class WorkerThread extends Thread { + + public WorkerThread(Runnable r) { + super(r); + } + + @Override + public void run() { + try { + super.run(); + } catch (Exception e) { + log.error(String.format("Error while sending logger entry: %s", e), e); + } + } + + } + + private static class WorkerThreadExecutor implements Executor { + + protected final ThreadPoolExecutor executor; + + WorkerThreadExecutor(int threads, int queueSize) { + this(threads, queueSize, WorkerThread::new); + } + + WorkerThreadExecutor(int threads, int queueSize, ThreadFactory threadFactory) { + executor = new ThreadPoolExecutor( + threads, threads, + 0L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(queueSize), + threadFactory); + } + + public void close() { + try { + executor.shutdown(); + executor.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // + } finally { + executor.shutdownNow(); + } + } + + @Override + public void execute(Runnable r) { + executor.execute(r); + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/logging/LocalDiskLogger.java b/container-search/src/main/java/com/yahoo/search/logging/LocalDiskLogger.java new file mode 100644 index 00000000000..6d815059066 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/logging/LocalDiskLogger.java @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.logging; + +import java.io.FileWriter; +import java.io.IOException; + +public class LocalDiskLogger extends AbstractThreadedLogger { + + private String logFilePath; + + public LocalDiskLogger(LocalDiskLoggerConfig config) { + logFilePath = config.path(); + } + + @Override + void transport(LoggerEntry entry) { + String json = entry.toJson(); + try (FileWriter fw = new FileWriter(logFilePath, true)) { + fw.write(json); + fw.write(System.getProperty("line.separator")); + } catch (IOException e) { + e.printStackTrace(); + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/logging/Logger.java b/container-search/src/main/java/com/yahoo/search/logging/Logger.java new file mode 100644 index 00000000000..3938b01c7b7 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/logging/Logger.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.logging; + +public interface Logger { + + default LoggerEntry.Builder newEntry() { + return new LoggerEntry.Builder(this); + } + + boolean send(LoggerEntry entry); + +} diff --git a/container-search/src/main/java/com/yahoo/search/logging/LoggerEntry.java b/container-search/src/main/java/com/yahoo/search/logging/LoggerEntry.java new file mode 100644 index 00000000000..44eec6e64a0 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/logging/LoggerEntry.java @@ -0,0 +1,111 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.logging; + +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.search.Query; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.Base64; + +public class LoggerEntry { + + private final Long timestamp; + private final Query query; + private final ByteBuffer blob; + + private LoggerEntry(Builder builder) { + timestamp = builder.timestamp; // or set automatically if not set + query = builder.query; + blob = builder.blob; + } + + public Long timestamp() { + return timestamp; + } + + public Query query() { + return query; + } + + public String queryString() { + String queryString = null; + if (query != null) { + if (query.getHttpRequest() != null && query.getHttpRequest().getUri() != null) { + queryString = query.getHttpRequest().getUri().getPath(); + if (query.getHttpRequest().getUri().getQuery() != null) { + queryString += "?" + query.getHttpRequest().getUri().getRawQuery(); + } + } + } + return queryString; + } + + public ByteBuffer blob() { + return blob; + } + + public String toString() { + return toJson(); + } + + public String toJson() { + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + JsonGenerator g = new JsonFactory().createGenerator(out, JsonEncoding.UTF8); + g.writeStartObject(); + + g.writeNumberField("timestamp", timestamp == null ? 0 : timestamp); + g.writeStringField("query", queryString()); + g.writeStringField("blob", Base64.getEncoder().encodeToString(blob.array())); + + g.writeEndObject(); + g.close(); + return out.toString(); + + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static class Builder { + + private final Logger logger; + + private Long timestamp; + private Query query; + private ByteBuffer blob; + + public Builder(Logger logger) { + this.logger = logger; + } + + public Builder timestamp(long timestamp) { + this.timestamp = timestamp; + return this; + } + + public Builder query(Query query) { + this.query = query; + return this; + } + + public Builder blob(byte[] bytes) { + blob = ByteBuffer.allocate(bytes.length); + blob.put(bytes).limit(blob.position()).position(0); + return this; + } + + public boolean send() { + return logger.send(new LoggerEntry(this)); + } + + } + + +} diff --git a/container-search/src/main/java/com/yahoo/search/logging/package-info.java b/container-search/src/main/java/com/yahoo/search/logging/package-info.java new file mode 100644 index 00000000000..e2bc14f4faf --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/logging/package-info.java @@ -0,0 +1,6 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +@ExportPackage +package com.yahoo.search.logging; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/resources/configdefinitions/search.logging.local-disk-logger.def b/container-search/src/main/resources/configdefinitions/search.logging.local-disk-logger.def new file mode 100644 index 00000000000..fc42c0dbd20 --- /dev/null +++ b/container-search/src/main/resources/configdefinitions/search.logging.local-disk-logger.def @@ -0,0 +1,5 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=search.logging + +## Path where logs should be put +path string default=/dev/null diff --git a/container-search/src/test/java/com/yahoo/search/logging/LocalDiskLoggerTest.java b/container-search/src/test/java/com/yahoo/search/logging/LocalDiskLoggerTest.java new file mode 100644 index 00000000000..bf6938424d7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/logging/LocalDiskLoggerTest.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.logging; + +import com.yahoo.io.IOUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Base64; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +public class LocalDiskLoggerTest { + + @TempDir + Path tempDir; + + @Test + public void testLocalDiskLogger() throws InterruptedException, IOException { + File logFile = tempDir.resolve("localdisklogger.log").toFile(); + + LocalDiskLoggerConfig.Builder builder = new LocalDiskLoggerConfig.Builder(); + builder.path(logFile.getAbsolutePath()); + LocalDiskLogger logger = new LocalDiskLogger(builder.build()); + + logger.newEntry() + .blob("Yo entry".getBytes()) + .send(); + waitForFile(logFile); + + String test = IOUtils.readAll(new FileReader(logFile)); + assertTrue(test.contains(Base64.getEncoder().encodeToString("Yo entry".getBytes()))); + } + + private void waitForFile(File file) throws InterruptedException { + int waitFor = 10; + while ( ! file.exists() && --waitFor > 0) { + Thread.sleep(10); + } + if ( ! file.exists()) { + fail("Local disk logger file was not created"); + } + } + +} |