diff options
author | Arne Juul <arnej@yahoo-inc.com> | 2018-09-03 08:00:12 +0000 |
---|---|---|
committer | Arne Juul <arnej@yahoo-inc.com> | 2018-09-06 13:00:14 +0000 |
commit | 30fd1322c666d8e48fae340dc69ce5030069e30b (patch) | |
tree | ee2b340ea309ff07d8bc867952dc65abed863f06 /vespalog | |
parent | ae594f2b7453ff1c5109fd8a9cec9339ec0a6366 (diff) |
enforce log retention policies
* for access logs, save meta-data about the log file itself
in a simple format.
* implement a proof-of-concept shell script that removes
log files after one month.
* ensure retention enforcer is started when services start
* note that retention enforcer will continue running even
after services stop, but it has protection to ensure that
it won't multiply endlessly.
Diffstat (limited to 'vespalog')
-rw-r--r-- | vespalog/pom.xml | 4 | ||||
-rw-r--r-- | vespalog/src/main/java/com/yahoo/log/LogFileDb.java | 50 | ||||
-rw-r--r-- | vespalog/src/test/java/com/yahoo/log/LogFileDbTest.java | 29 |
3 files changed, 83 insertions, 0 deletions
diff --git a/vespalog/pom.xml b/vespalog/pom.xml index 6443769afbe..7b167ee2c1c 100644 --- a/vespalog/pom.xml +++ b/vespalog/pom.xml @@ -50,6 +50,10 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <configuration> + <forkMode>once</forkMode> + <environmentVariables> + <VESPA_HOME>${project.build.directory}</VESPA_HOME> + </environmentVariables> <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> </configuration> </plugin> diff --git a/vespalog/src/main/java/com/yahoo/log/LogFileDb.java b/vespalog/src/main/java/com/yahoo/log/LogFileDb.java new file mode 100644 index 00000000000..d0fa64805bf --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/LogFileDb.java @@ -0,0 +1,50 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardOpenOption.*; + +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static com.yahoo.vespa.defaults.Defaults.getDefaults; + + +/** + * @author arnej + * + * This class takes care of saving meta-data about a log-file, + * ensuring that we can enact policies about log retention. + **/ +public class LogFileDb { + + static final String DBDIR = "var/db/vespa/logfiledb/"; + + private static long dayStamp() { + long s = System.currentTimeMillis() / 1000; + return s / 100000; + } + + private static OutputStream metaFile() throws java.io.IOException { + String fn = getDefaults().underVespaHome(DBDIR + "logfiles." + dayStamp()); + Path path = Paths.get(fn); + return Files.newOutputStream(path, CREATE, APPEND); + } + + public static void nowLoggingTo(String filename) { + if (filename.contains("\n")) { + throw new IllegalArgumentException("Cannot use filename with newline: "+filename); + } + long s = System.currentTimeMillis() / 1000; + String meta = "" + s + " " + filename + "\n"; + byte[] data = meta.getBytes(UTF_8); + try (OutputStream out = metaFile()) { + out.write(data); + } catch (java.io.IOException e) { + System.err.println("Saving meta-data about logfile "+filename+" failed: "+e); + // ignore + } + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/LogFileDbTest.java b/vespalog/src/test/java/com/yahoo/log/LogFileDbTest.java new file mode 100644 index 00000000000..4dd7bd0978c --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/LogFileDbTest.java @@ -0,0 +1,29 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.File; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import org.junit.Test; + +/** + * @author arnej + */ +public class LogFileDbTest { + + @Test + public void canSave() { + System.err.println("VH: "+System.getenv("VESPA_HOME")); + File dir = new File(getDefaults().underVespaHome(LogFileDb.DBDIR)); + dir.mkdirs(); + if (dir.isDirectory()) { + System.err.println("using directory: "+dir); + new File(getDefaults().underVespaHome("logs/extra")).mkdirs(); + String fn = getDefaults().underVespaHome("logs/extra/foo-bar.log"); + LogFileDb.nowLoggingTo(fn); + fn = getDefaults().underVespaHome("logs/extra/stamped-1.log"); + LogFileDb.nowLoggingTo(fn); + } else { + System.err.println("cannot create directory: "+dir); + } + } +} |