From 30fd1322c666d8e48fae340dc69ce5030069e30b Mon Sep 17 00:00:00 2001
From: Arne Juul <arnej@yahoo-inc.com>
Date: Mon, 3 Sep 2018 08:00:12 +0000
Subject: enforce log retention policies

* for access logs, save meta-data about the log file itself
  in a simple format.
* implement a proof-of-concept shell script that removes
  log files after one month.
* ensure retention enforcer is started when services start
* note that retention enforcer will continue running even
  after services stop, but it has protection to ensure that
  it won't multiply endlessly.
---
 vespalog/pom.xml                                   |  4 ++
 .../src/main/java/com/yahoo/log/LogFileDb.java     | 50 ++++++++++++++++++++++
 .../src/test/java/com/yahoo/log/LogFileDbTest.java | 29 +++++++++++++
 3 files changed, 83 insertions(+)
 create mode 100644 vespalog/src/main/java/com/yahoo/log/LogFileDb.java
 create mode 100644 vespalog/src/test/java/com/yahoo/log/LogFileDbTest.java

(limited to 'vespalog')

diff --git a/vespalog/pom.xml b/vespalog/pom.xml
index 6443769afbe..7b167ee2c1c 100644
--- a/vespalog/pom.xml
+++ b/vespalog/pom.xml
@@ -50,6 +50,10 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
+          <forkMode>once</forkMode>
+          <environmentVariables>
+            <VESPA_HOME>${project.build.directory}</VESPA_HOME>
+          </environmentVariables>
           <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile>
         </configuration>
       </plugin>
diff --git a/vespalog/src/main/java/com/yahoo/log/LogFileDb.java b/vespalog/src/main/java/com/yahoo/log/LogFileDb.java
new file mode 100644
index 00000000000..d0fa64805bf
--- /dev/null
+++ b/vespalog/src/main/java/com/yahoo/log/LogFileDb.java
@@ -0,0 +1,50 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.log;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.StandardOpenOption.*;
+
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+
+
+/**
+ * @author arnej
+ *
+ * This class takes care of saving meta-data about a log-file,
+ * ensuring that we can enact policies about log retention.
+ **/
+public class LogFileDb {
+
+    static final String DBDIR = "var/db/vespa/logfiledb/";
+
+    private static long dayStamp() {
+        long s = System.currentTimeMillis() / 1000;
+        return s / 100000;
+    }
+
+    private static OutputStream metaFile() throws java.io.IOException {
+        String fn = getDefaults().underVespaHome(DBDIR + "logfiles." + dayStamp());
+        Path path = Paths.get(fn);
+        return Files.newOutputStream(path, CREATE, APPEND);
+    }
+
+    public static void nowLoggingTo(String filename) {
+        if (filename.contains("\n")) {
+            throw new IllegalArgumentException("Cannot use filename with newline: "+filename);
+        }
+        long s = System.currentTimeMillis() / 1000;
+        String meta = "" + s + " " + filename + "\n";
+        byte[] data = meta.getBytes(UTF_8);
+        try (OutputStream out = metaFile()) {
+            out.write(data);
+        } catch (java.io.IOException e) {
+            System.err.println("Saving meta-data about logfile "+filename+" failed: "+e);
+            // ignore
+        }
+    }
+}
diff --git a/vespalog/src/test/java/com/yahoo/log/LogFileDbTest.java b/vespalog/src/test/java/com/yahoo/log/LogFileDbTest.java
new file mode 100644
index 00000000000..4dd7bd0978c
--- /dev/null
+++ b/vespalog/src/test/java/com/yahoo/log/LogFileDbTest.java
@@ -0,0 +1,29 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.log;
+
+import java.io.File;
+import static com.yahoo.vespa.defaults.Defaults.getDefaults;
+import org.junit.Test;
+
+/**
+ * @author arnej
+ */
+public class LogFileDbTest {
+
+    @Test
+    public void canSave() {
+        System.err.println("VH: "+System.getenv("VESPA_HOME"));
+        File dir = new File(getDefaults().underVespaHome(LogFileDb.DBDIR));
+        dir.mkdirs();
+        if (dir.isDirectory()) {
+            System.err.println("using directory: "+dir);
+            new File(getDefaults().underVespaHome("logs/extra")).mkdirs();
+            String fn = getDefaults().underVespaHome("logs/extra/foo-bar.log");
+            LogFileDb.nowLoggingTo(fn);
+            fn = getDefaults().underVespaHome("logs/extra/stamped-1.log");
+            LogFileDb.nowLoggingTo(fn);
+        } else {
+            System.err.println("cannot create directory: "+dir);
+        }
+    }
+}
-- 
cgit v1.2.3