enforce log retention policies

* for access logs, save meta-data about the log file itself
  in a simple format.
* implement a proof-of-concept shell script that removes
  log files after one month.
* ensure retention enforcer is started when services start
* note that retention enforcer will continue running even
  after services stop, but it has protection to ensure that
  it won't multiply endlessly.

1 files changed, 50 insertions, 0 deletions

diff --git a/vespalog/src/main/java/com/yahoo/log/LogFileDb.java b/vespalog/src/main/java/com/yahoo/log/LogFileDb.java
new file mode 100644
index 00000000000..d0fa64805bf
--- /dev/null
+++ b/vespalog/src/main/java/com/yahoo/log/LogFileDb.java
@@ -0,0 +1,50 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.log;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.StandardOpenOption.*;
+
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+
+
+/**
+ * @author arnej
+ *
+ * This class takes care of saving meta-data about a log-file,
+ * ensuring that we can enact policies about log retention.
+ **/
+public class LogFileDb {
+
+    static final String DBDIR = "var/db/vespa/logfiledb/";
+
+    private static long dayStamp() {
+        long s = System.currentTimeMillis() / 1000;
+        return s / 100000;
+    }
+
+    private static OutputStream metaFile() throws java.io.IOException {
+        String fn = getDefaults().underVespaHome(DBDIR + "logfiles." + dayStamp());
+        Path path = Paths.get(fn);
+        return Files.newOutputStream(path, CREATE, APPEND);
+    }
+
+    public static void nowLoggingTo(String filename) {
+        if (filename.contains("\n")) {
+            throw new IllegalArgumentException("Cannot use filename with newline: "+filename);
+        }
+        long s = System.currentTimeMillis() / 1000;
+        String meta = "" + s + " " + filename + "\n";
+        byte[] data = meta.getBytes(UTF_8);
+        try (OutputStream out = metaFile()) {
+            out.write(data);
+        } catch (java.io.IOException e) {
+            System.err.println("Saving meta-data about logfile "+filename+" failed: "+e);
+            // ignore
+        }
+    }
+}