Track when file reference or download was last requested

3 files changed, 48 insertions, 22 deletions

diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java
index 33a8ed405a9..0d72a1f02b6 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java
@@ -29,7 +29,7 @@ import java.util.stream.Collectors;
  *
  * @author hmusum
  */
-public class FileDistributionRpcServer {
+class FileDistributionRpcServer {
 
     private final static Logger log = Logger.getLogger(FileDistributionRpcServer.class.getName());
 
@@ -38,13 +38,13 @@ public class FileDistributionRpcServer {
     private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(),
                                                                                      new DaemonThreadFactory("Rpc executor"));
 
-    public FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) {
+    FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) {
         this.supervisor = supervisor;
         this.downloader = downloader;
         declareFileDistributionMethods();
     }
 
-    public void close() {
+    void close() {
         rpcDownloadExecutor.shutdownNow();
         try {
             rpcDownloadExecutor.awaitTermination(10, TimeUnit.SECONDS);
@@ -80,8 +80,6 @@ public class FileDistributionRpcServer {
     private static final int baseFileProviderErrorCode = baseErrorCode + 0x1000;
 
     private static final int fileReferenceDoesNotExists = baseFileProviderErrorCode;
-    private static final int fileReferenceRemoved = fileReferenceDoesNotExists + 1;
-    private static final int fileReferenceInternalError = fileReferenceRemoved + 1;
 
     private void getFile(Request req) {
         req.detach();
@@ -118,19 +116,16 @@ public class FileDistributionRpcServer {
     private void downloadFile(Request req) {
         FileReference fileReference = new FileReference(req.parameters().get(0).asString());
         log.log(LogLevel.DEBUG, () -> "getFile() called for file reference '" + fileReference.value() + "'");
-        Optional<File> pathToFile = downloader.getFile(fileReference);
-        try {
-            if (pathToFile.isPresent()) {
-                req.returnValues().add(new StringValue(pathToFile.get().getAbsolutePath()));
-                log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' available at " + pathToFile.get());
-            } else {
-                log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error");
-                req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found");
-            }
-        } catch (Throwable e) {
-            log.log(LogLevel.WARNING, "File reference '" + fileReference.value() + "' got exception: " + e.getMessage());
-            req.setError(fileReferenceInternalError, "File reference '" + fileReference.value() + "' removed");
+        Optional<File> file = downloader.getFile(fileReference);
+        if (file.isPresent()) {
+            new RequestTracker().trackRequest(file.get().getParentFile());
+            req.returnValues().add(new StringValue(file.get().getAbsolutePath()));
+            log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' available at " + file.get());
+        } else {
+            log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error");
+            req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found");
         }
+
         req.returnRequest();
     }
 
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java
new file mode 100644
index 00000000000..47f478ea4d7
--- /dev/null
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java
@@ -0,0 +1,30 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.vespa.config.proxy.filedistribution;
+
+import com.yahoo.log.LogLevel;
+
+import java.io.File;
+import java.time.Instant;
+import java.util.logging.Logger;
+
+/**
+ * Set last modification time for a file reference or downloaded url, to be able
+ * to later clean up file references or urls not used for a long time.
+ *
+ * @author hmusum
+ */
+class RequestTracker {
+
+    private final static Logger log = Logger.getLogger(RequestTracker.class.getName());
+
+    void trackRequest(File file) {
+        String absolutePath = file.getAbsolutePath();
+        if ( ! file.exists())
+            log.log(LogLevel.WARNING, "Could not find file '" + absolutePath + "'");
+
+        if ( ! file.setLastModified(Instant.now().toEpochMilli()))
+            log.log(LogLevel.WARNING, "Could not set last modified timestamp for '" + absolutePath + "'");
+    }
+
+}
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java
index 9d89f1d10b2..cdf079631fe 100644
--- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java
+++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java
@@ -42,7 +42,7 @@ class UrlDownloadRpcServer {
     private final static Logger log = Logger.getLogger(UrlDownloadRpcServer.class.getName());
 
     private static final String CONTENTS_FILE_NAME = "contents";
-    private static final String LAST_MODFIED_FILE_NAME = "lastmodified";
+    private static final String LAST_MODIFIED_FILE_NAME = "lastmodified";
 
     private final File downloadBaseDir;
     private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()),
@@ -110,16 +110,17 @@ class UrlDownloadRpcServer {
 
                 if (contentsPath.exists() && contentsPath.length() > 0) {
                     writeLastModifiedTimestamp(downloadDir, connection.getLastModified());
+                    new RequestTracker().trackRequest(downloadDir);
                     req.returnValues().add(new StringValue(contentsPath.getAbsolutePath()));
                     log.log(LogLevel.DEBUG, () -> "URL '" + url + "' available at " + contentsPath);
+                    log.log(LogLevel.INFO, String.format("Download of URL '%s' done in %.3f seconds",
+                                                         url, (System.currentTimeMillis() -start) / 1000.0));
                 } else {
                     log.log(LogLevel.ERROR, "Downloaded URL '" + url + "' not found, returning error");
                     req.setError(DOES_NOT_EXIST, "Downloaded '" + url + "' not found");
                 }
             }
         }
-        long end = System.currentTimeMillis();
-        log.log(LogLevel.INFO, String.format("Download of URL '%s' done in %.3f seconds", url,  (end-start) / 1000.0));
     }
 
     private static String urlToDirName(String uri) {
@@ -137,7 +138,7 @@ class UrlDownloadRpcServer {
     }
 
     private static long readLastModifiedTimestamp(File downloadDir) throws IOException {
-        File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME);
+        File lastModified = new File(downloadDir, LAST_MODIFIED_FILE_NAME);
         if (lastModified.exists() && lastModified.length() > 0) {
             try (BufferedReader br = new BufferedReader(new FileReader(lastModified))) {
                 String timestamp = br.readLine();
@@ -148,7 +149,7 @@ class UrlDownloadRpcServer {
     }
 
     private static void writeLastModifiedTimestamp(File downloadDir, long timestamp) throws IOException {
-        File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME);
+        File lastModified = new File(downloadDir, LAST_MODIFIED_FILE_NAME);
         try (BufferedWriter lastModifiedWriter = new BufferedWriter(new FileWriter(lastModified.getAbsolutePath()))) {
             lastModifiedWriter.write(Long.toString(timestamp));
         }