Revert "Revert "Balder/add native fadvise""

5 files changed, 148 insertions, 4 deletions

diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
index d729b092670..9963429bf97 100644
--- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
+++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java
@@ -2,6 +2,7 @@
 package com.yahoo.container.logging;
 
 import com.yahoo.container.core.AccessLogConfig;
+import com.yahoo.io.NativeIO;
 import com.yahoo.log.LogFileDb;
 
 import java.io.File;
@@ -263,15 +264,28 @@ public class LogFileHandler extends StreamHandler {
         numberOfRecords = 0;
         lastRotationTime = now;
         nextRotationTime = 0; //figure it out later (lazy evaluation)
-        if (compressOnRotation && (oldFileName != null)) {
-            triggerCompression(oldFileName);
+        if ((oldFileName != null)) {
+            if (compressOnRotation) {
+                triggerCompression(oldFileName);
+            } else {
+                NativeIO nativeIO = new NativeIO();
+                nativeIO.dropFileFromCache(new File(oldFileName));
+            }
         }
     }
 
     private void triggerCompression(String oldFileName) {
         try {
+            String gzippedFileName = oldFileName + ".gz";
             Runtime r = Runtime.getRuntime();
-            Process p = r.exec(new String[] { "gzip", oldFileName });
+            StringBuilder cmd = new StringBuilder("gzip");
+            cmd.append(" < "). append(oldFileName).append(" > ").append(gzippedFileName);
+            Process p = r.exec(cmd.toString());
+            NativeIO nativeIO = new NativeIO();
+            File oldFile = new File(oldFileName);
+            nativeIO.dropFileFromCache(oldFile); // Drop from cache in case somebody else has a reference to it preventing from dying quickly.
+            oldFile.delete();
+            nativeIO.dropFileFromCache(new File(gzippedFileName));
             // Detonator pattern: Think of all the fun we can have if gzip isn't what we
             // think it is, if it doesn't return, etc, etc
         } catch (IOException e) {
diff --git a/parent/pom.xml b/parent/pom.xml
index 96075974567..a46b86f2725 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -461,7 +461,7 @@
                 <artifactId>commons-pool</artifactId>
                 <version>1.5.6</version>
             </dependency>
-            <!-- Explicitly included to get Zookeeper version 3.4.12,
+            <!-- Explicitly included to get Zookeeper version 3.4.13,
                  can be excluded if you want the Zookeeper version
                  used by curator by default
             -->
@@ -673,6 +673,11 @@
                 <artifactId>language-detector</artifactId>
                 <version>0.6</version>
             </dependency>
+            <dependency>
+                <groupId>net.java.dev.jna</groupId>
+                <artifactId>jna</artifactId>
+                <version>4.5.2</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml
index 5b9c143a447..741b4b58cc9 100644
--- a/vespajlib/pom.xml
+++ b/vespajlib/pom.xml
@@ -31,6 +31,10 @@
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-exec</artifactId>
     </dependency>
+    <dependency>
+      <groupId>net.java.dev.jna</groupId>
+      <artifactId>jna</artifactId>
+    </dependency>
 
 
     <!-- provided scope -->
diff --git a/vespajlib/src/main/java/com/yahoo/io/NativeIO.java b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java
new file mode 100644
index 00000000000..bee43a8653e
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java
@@ -0,0 +1,87 @@
+package com.yahoo.io;
+
+import java.io.File;
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.logging.Logger;
+
+import com.sun.jna.LastErrorException;
+import com.sun.jna.Native;
+import com.sun.jna.Platform;
+
+public class NativeIO {
+    private final Logger logger = Logger.getLogger(getClass().getName());
+    private static final int POSIX_FADV_DONTNEED = 4; // See /usr/include/linux/fadvise.h
+    private static boolean initialized = false;
+    private static Throwable initError = null;
+    static {
+        try {
+            if (Platform.isLinux()) {
+                Native.register(Platform.C_LIBRARY_NAME);
+                initialized = true;
+            }
+        } catch (Throwable throwable) {
+            initError = throwable;
+        }
+    }
+
+    private static final Field fieldFD = getField(FileDescriptor.class, "fd");
+
+
+    private static native int posix_fadvise(int fd, long offset, long len, int flag) throws LastErrorException;
+
+    public NativeIO() {
+        if (!initialized) {
+            logger.warning("native IO not possible due to " + getError().getMessage());
+        }
+    }
+
+    public boolean valid() { return initialized; }
+    public Throwable getError() {
+        if (initError != null) {
+            return initError;
+        } else {
+            return new RuntimeException("Platform is unsúpported. Only supported on linux.");
+        }
+    }
+
+    public void dropFileFromCache(FileDescriptor fd) {
+        if (initialized) {
+            posix_fadvise(getNativeFD(fd), 0, 0, POSIX_FADV_DONTNEED);
+        }
+    }
+
+    public void dropFileFromCache(File file) {
+        try {
+            dropFileFromCache(new FileInputStream(file).getFD());
+        } catch (FileNotFoundException e) {
+            throw new RuntimeException(e);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    private static Field getField(Class<?> clazz, String fieldName) {
+        Field field;
+        try {
+            field = clazz.getDeclaredField(fieldName);
+        } catch (NoSuchFieldException e) {
+            throw new RuntimeException(e);
+        }
+        field.setAccessible(true);
+        return field;
+    }
+
+    private static int getNativeFD(FileDescriptor fd) {
+        try {
+            return fieldFD.getInt(fd);
+        } catch (IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}
diff --git a/vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java b/vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java
new file mode 100644
index 00000000000..ecd38056a19
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java
@@ -0,0 +1,34 @@
+package com.yahoo.io;
+
+import com.sun.jna.Platform;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+public class NativeIOTestCase {
+
+    @Test
+    public void requireThatDropFileFromCacheDoesNotThrow() throws IOException {
+        File testFile = new File("testfile");
+        FileOutputStream output = new FileOutputStream(testFile);
+        output.write('t');
+        output.flush();
+        output.close();
+        NativeIO nativeIO = new NativeIO();
+        if (Platform.isLinux()) {
+            assertTrue(nativeIO.valid());
+        } else {
+            assertFalse(nativeIO.valid());
+            assertEquals("Platform is unsúpported. Only supported on linux.", nativeIO.getError().getMessage());
+        }
+        nativeIO.dropFileFromCache(output.getFD());
+        nativeIO.dropFileFromCache(testFile);
+        testFile.delete();
+    }
+}