enforce log retention policies

* for access logs, save meta-data about the log file itself
  in a simple format.
* implement a proof-of-concept shell script that removes
  log files after one month.
* ensure retention enforcer is started when services start
* note that retention enforcer will continue running even
  after services stop, but it has protection to ensure that
  it won't multiply endlessly.

2 files changed, 12 insertions, 2 deletions

diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
index c7e2a777695..d729b092670 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
@@ -2,6 +2,7 @@
 package com.yahoo.container.logging;
 
 import com.yahoo.container.core.AccessLogConfig;
+import com.yahoo.log.LogFileDb;
 
 import java.io.File;
 import java.io.FileOutputStream;
@@ -250,6 +251,7 @@ public class LogFileHandler extends StreamHandler {
             FileOutputStream os = new FileOutputStream(fileName, true); // append mode, for safety
             super.setOutputStream(os);
             currentOutputStream = os;
+            if (! useSequenceNameScheme) LogFileDb.nowLoggingTo(fileName);
         }
         catch (IOException e) {
             throw new RuntimeException("Couldn't open log file '" + fileName + "'", e);
@@ -310,7 +312,9 @@ public class LogFileHandler extends StreamHandler {
             if (thisN>largestN)
                 largestN=thisN;
         }
-        file.renameTo(new File(dir,file.getName() + "." + (largestN + 1)));
+        File newFn = new File(dir, file.getName() + "." + (largestN + 1));
+        LogFileDb.nowLoggingTo(newFn.getAbsolutePath());
+        file.renameTo(newFn);
     }
 
     /**
diff --git a/container-accesslogging/src/main/resources/configdefinitions/access-log.def b/container-accesslogging/src/main/resources/configdefinitions/access-log.def
index 276128e0405..9df9299ae19 100644
--- a/container-accesslogging/src/main/resources/configdefinitions/access-log.def
+++ b/container-accesslogging/src/main/resources/configdefinitions/access-log.def
@@ -1,11 +1,16 @@
 # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 namespace=container.core
 
-
 # File name patterns supporting the expected time variables, e.g. ".%Y%m%d%H%M%S"
 fileHandler.pattern string
+
+# When should rotation happen, in minutes after midnight
+# Does this really need to be configurable?
+# Could just configure "every N minutes" instead
 fileHandler.rotation string default="0 60 ..."
 
+# TODO remove in Vespa 7, always use DATE
+#
 # Defines how file rotation is done. There are two options:
 #
 # DATE:
@@ -27,4 +32,5 @@ fileHandler.rotateScheme enum {DATE, SEQUENCE} default=DATE
 fileHandler.symlink string default=""
 
 # compress the previous access log after rotation
+# TODO change to "true" for Vespa 7
 fileHandler.compressOnRotation bool default=false