Add maintainer for deleting unused file references on disk

* For now the maintainer just outputs what to be deleted
* Add config for file references dir insteaed of hardcoding
  to make it possible to test

21 files changed, 237 insertions, 66 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java
index ac1bcfa542a..9b457f49bd2 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java
@@ -23,12 +23,18 @@ public interface FileDistribution {
      */
     void startDownload(String hostName, int port, Set<FileReference> fileReferences);
 
+    // TODO: Remove when 6.244 is oldest version in use
+    @Deprecated
     static String getDefaultFileDBRoot() {
         return Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution");
     }
 
+    // TODO: Remove when 6.244 is oldest version in use
+    @Deprecated
     static File getDefaultFileDBPath() {
         return new File(getDefaultFileDBRoot());
     }
 
+    File getFileReferencesDir();
+
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
index eabd0e5a7e0..131a5344116 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java
@@ -7,10 +7,10 @@ import com.yahoo.config.model.application.provider.MockFileRegistry;
 import com.yahoo.config.model.test.MockHosts;
 import org.junit.Test;
 
+import java.io.File;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -48,5 +48,10 @@ public class FileDistributorTestCase {
         public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {
             filesToDownloadCalled++;
         }
+
+        @Override
+        public File getFileReferencesDir() {
+            return null;
+        }
     }
 }
diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def
index 5e81526dc53..bf4c9599f4a 100644
--- a/configdefinitions/src/vespa/configserver.def
+++ b/configdefinitions/src/vespa/configserver.def
@@ -13,10 +13,13 @@ zookeeperserver[].port int default=2181
 zookeeper.barrierTimeout long default=120 # in seconds
 zookeeperLocalhostAffinity bool default=true
 
-# Misc
+# Directories
 configModelPluginDir[] string
 configServerDBDir string default="var/db/vespa/config_server/serverdb/"
 configDefinitionsDir string default="share/vespa/configdefinitions/"
+fileReferencesDir string default="var/db/vespa/filedistribution/"
+
+# Misc
 sessionLifetime long default=3600 # in seconds
 masterGeneration long default=0
 multitenant bool default=false
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
index 102a36e833f..07809feb057 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java
@@ -7,6 +7,7 @@ import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
 import com.yahoo.concurrent.DaemonThreadFactory;
+import com.yahoo.config.FileReference;
 import com.yahoo.config.application.api.ApplicationFile;
 import com.yahoo.config.application.api.ApplicationMetaData;
 import com.yahoo.config.application.api.DeployLogger;
@@ -54,7 +55,9 @@ import java.net.URI;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -269,29 +272,61 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return true;
     }
 
-    public HttpResponse clusterControllerStatusPage(Tenant tenant, ApplicationId applicationId, String hostName, String pathSuffix) {
-        Application application = getApplication(tenant, applicationId);
-
+    public HttpResponse clusterControllerStatusPage(ApplicationId applicationId, String hostName, String pathSuffix) {
         // WARNING: pathSuffix may be given by the external user. Make sure no security issues arise...
         // We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise
         // change the hostname and port. Exposing other paths on the cluster controller should be fine.
         // TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components.
         String relativePath = "clustercontroller-status/" + pathSuffix;
 
-        return httpProxy.get(application, hostName, "container-clustercontroller", relativePath);
+        return httpProxy.get(getApplication(applicationId), hostName, "container-clustercontroller", relativePath);
     }
 
-    public Long getApplicationGeneration(Tenant tenant, ApplicationId applicationId) {
-        return getApplication(tenant, applicationId).getApplicationGeneration();
+    public Long getApplicationGeneration(ApplicationId applicationId) {
+        return getApplication(applicationId).getApplicationGeneration();
     }
 
     public void restart(ApplicationId applicationId, HostFilter hostFilter) {
         hostProvisioner.ifPresent(provisioner -> provisioner.restart(applicationId, hostFilter));
     }
 
-    public HttpResponse filedistributionStatus(Tenant tenant, ApplicationId applicationId, Duration timeout) {
-        Application application = getApplication(tenant, applicationId);
-        return fileDistributionStatus.status(application, timeout);
+    public HttpResponse filedistributionStatus(ApplicationId applicationId, Duration timeout) {
+        return fileDistributionStatus.status(getApplication(applicationId), timeout);
+    }
+
+    public Set<String> deleteUnusedFiledistributionReferences(File fileReferencesPath, boolean deleteFromDisk) {
+        // Find all file references in use
+        Set<String> fileReferencesInUse = new HashSet<>();
+        Set<ApplicationId> applicationIds = listApplications();
+        applicationIds.forEach(applicationId -> fileReferencesInUse.addAll(getApplication(applicationId).getModel().fileReferences()
+                                           .stream()
+                                           .map(FileReference::value)
+                                           .collect(Collectors.toSet())));
+        log.log(LogLevel.INFO, "File references in use : " + fileReferencesInUse);
+
+        // Find those on disk that are not in use
+        if (!fileReferencesPath.isDirectory())
+            throw new RuntimeException(fileReferencesPath + " is not a directory");
+        Set<String> fileReferencesOnDisk = new HashSet<>();
+        File[] filesOnDisk = fileReferencesPath.listFiles();
+        if (filesOnDisk != null)
+            fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet()));
+        log.log(LogLevel.INFO, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk);
+
+        // TODO: Only consider the ones modified more than some time (14 days?) ago
+        Set<String> fileReferencesToDelete = fileReferencesOnDisk
+                .stream()
+                .filter(fileReference -> ! fileReferencesInUse.contains(fileReference))
+                .collect(Collectors.toSet());
+        if (deleteFromDisk) {
+            log.log(LogLevel.INFO, "Will delete file references not in use: " + fileReferencesToDelete);
+            fileReferencesToDelete.forEach(fileReference -> {
+                File file = new File(fileReferencesPath, fileReference);
+                if ( ! IOUtils.recursiveDeleteDir(file))
+                    log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath());
+            });
+        }
+        return fileReferencesToDelete;
     }
 
     public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, LocalSession.Mode mode) {
@@ -299,22 +334,27 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye
         return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode);
     }
 
-    private Application getApplication(Tenant tenant, ApplicationId applicationId) {
+    private Application getApplication(ApplicationId applicationId) {
+        Tenant tenant = tenantRepository.getTenant(applicationId.tenant());
         long sessionId = getSessionIdForApplication(tenant, applicationId);
         RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId, 0);
         return session.ensureApplicationLoaded().getForVersionOrLatest(Optional.empty(), clock.instant());
     }
 
+    private Set<ApplicationId> listApplications() {
+        return tenantRepository.getAllTenants().stream()
+                .flatMap(tenant -> tenant.getApplicationRepo().listApplications().stream())
+                .collect(Collectors.toSet());
+    }
+
     // ---------------- Convergence ----------------------------------------------------------------
 
-    public HttpResponse serviceConvergenceCheck(Tenant tenant, ApplicationId applicationId, String hostname, URI uri) {
-        Application application = getApplication(tenant, applicationId);
-        return convergeChecker.serviceConvergenceCheck(application, hostname, uri);
+    public HttpResponse serviceConvergenceCheck(ApplicationId applicationId, String hostname, URI uri) {
+        return convergeChecker.serviceConvergenceCheck(getApplication(applicationId), hostname, uri);
     }
 
-    public HttpResponse serviceListToCheckForConfigConvergence(Tenant tenant, ApplicationId applicationId, URI uri) {
-        Application application = getApplication(tenant, applicationId);
-        return convergeChecker.serviceListToCheckForConfigConvergence(application, uri);
+    public HttpResponse serviceListToCheckForConfigConvergence(ApplicationId applicationId, URI uri) {
+        return convergeChecker.serviceListToCheckForConfigConvergence(getApplication(applicationId), uri);
     }
 
     // ---------------- Session operations ----------------------------------------------------------------
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
index 1c2c24cc7bb..df2287c64cb 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
@@ -26,10 +26,6 @@ public class FileDirectory  {
     private static final Logger log = Logger.getLogger(FileDirectory.class.getName());
     private final File root;
 
-    public FileDirectory() {
-        this(FileDistribution.getDefaultFileDBPath());
-    }
-
     public FileDirectory(File rootDir) {
         root = rootDir;
         try {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java
index 544451b8e10..2db89c2e8ed 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.filedistribution;
 
+import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.model.api.FileDistribution;
 import com.yahoo.jrt.ErrorCode;
@@ -11,7 +12,9 @@ import com.yahoo.jrt.Supervisor;
 import com.yahoo.jrt.Target;
 import com.yahoo.jrt.Transport;
 import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.defaults.Defaults;
 
+import java.io.File;
 import java.util.Set;
 import java.util.logging.Logger;
 
@@ -22,12 +25,22 @@ public class FileDistributionImpl implements FileDistribution {
     private final static Logger log = Logger.getLogger(FileDistributionImpl.class.getName());
 
     private final Supervisor supervisor = new Supervisor(new Transport());
+    private final File fileReferencesDir;
+
+    public FileDistributionImpl(ConfigserverConfig configserverConfig) {
+        this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()));
+    }
 
     @Override
     public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {
          startDownloadingFileReferences(hostName, port, fileReferences);
     }
 
+    @Override
+    public File getFileReferencesDir() {
+        return fileReferencesDir;
+    }
+
     // Notifies config proxy which file references it should start downloading. It's OK if the call does not succeed,
     // as downloading will then start synchronously when a service requests a file reference instead
     private void startDownloadingFileReferences(String hostName, int port, Set<FileReference> fileReferences) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
index d6751987424..b3f3214793c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java
@@ -17,8 +17,8 @@ public class FileDistributionProvider {
     private final FileDistribution fileDistribution;
 
     public FileDistributionProvider(File applicationDir, FileDistribution fileDistribution) {
-        this(new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory())), fileDistribution);
-        ensureDirExists(FileDistribution.getDefaultFileDBPath());
+        this(new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory(fileDistribution.getFileReferencesDir()))), fileDistribution);
+        ensureDirExists(fileDistribution.getFileReferencesDir());
     }
 
     FileDistributionProvider(FileRegistry fileRegistry, FileDistribution fileDistribution) {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
index 001ef751e69..42bf269e9d2 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.config.server.filedistribution;
 import com.google.inject.Inject;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.FileReference;
-import com.yahoo.config.model.api.FileDistribution;
 import com.yahoo.config.subscription.ConfigSourceSet;
 import com.yahoo.jrt.Int32Value;
 import com.yahoo.jrt.Request;
@@ -17,6 +16,7 @@ import com.yahoo.vespa.config.Connection;
 import com.yahoo.vespa.config.ConnectionPool;
 import com.yahoo.vespa.config.JRTConnectionPool;
 import com.yahoo.vespa.config.server.ConfigServerSpec;
+import com.yahoo.vespa.defaults.Defaults;
 import com.yahoo.vespa.filedistribution.CompressedFileReference;
 import com.yahoo.vespa.filedistribution.FileDownloader;
 import com.yahoo.vespa.filedistribution.FileReferenceData;
@@ -72,7 +72,7 @@ public class FileServer {
 
     @Inject
     public FileServer(ConfigserverConfig configserverConfig) {
-        this(createConnectionPool(configserverConfig), FileDistribution.getDefaultFileDBPath());
+        this(createConnectionPool(configserverConfig), new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())));
     }
 
     // For testing only
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java
index 728f327c829..40d75d9dbac 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java
@@ -4,14 +4,23 @@ package com.yahoo.vespa.config.server.filedistribution;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.model.api.FileDistribution;
 
+import java.io.File;
 import java.util.Set;
 
 /**
  * @author Ulf Lilleengen
  */
-public class MockFileDBHandler implements FileDistribution {
+public class MockFileDistribution implements FileDistribution {
+    private final File fileReferencesDir;
+
+    MockFileDistribution(File fileReferencesDir) {
+        this.fileReferencesDir = fileReferencesDir;
+    }
 
     @Override
     public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {}
 
+    @Override
+    public File getFileReferencesDir() { return fileReferencesDir; }
+
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java
index b4ed2352d00..db70a51b2b4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java
@@ -4,14 +4,16 @@ package com.yahoo.vespa.config.server.filedistribution;
 import com.yahoo.config.model.api.FileDistribution;
 import com.yahoo.config.model.application.provider.MockFileRegistry;
 
+import java.io.File;
+
 /**
  * @author Ulf Lilleengen
  */
 public class MockFileDistributionProvider extends FileDistributionProvider {
     public int timesCalled = 0;
 
-    public MockFileDistributionProvider() {
-        super(new MockFileRegistry(), new MockFileDBHandler());
+    public MockFileDistributionProvider(File fileReferencesDir) {
+        super(new MockFileRegistry(), new MockFileDistribution(fileReferencesDir));
     }
 
     public FileDistribution getFileDistribution() {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
index 42fdb16c7ca..6bca8b1c562 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java
@@ -59,13 +59,13 @@ public class ApplicationHandler extends HttpHandler {
         Tenant tenant = verifyTenantAndApplication(applicationId);
 
         if (isServiceConvergeRequest(request)) {
-            return applicationRepository.serviceConvergenceCheck(tenant, applicationId, getHostNameFromRequest(request), request.getUri());
+            return applicationRepository.serviceConvergenceCheck(applicationId, getHostNameFromRequest(request), request.getUri());
         }
 
         if (isClusterControllerStatusRequest(request)) {
             String hostName = getHostNameFromRequest(request);
             String pathSuffix = getPathSuffix(request);
-            return applicationRepository.clusterControllerStatusPage(tenant, applicationId, hostName, pathSuffix);
+            return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix);
         }
 
         if (isContentRequest(request)) {
@@ -86,15 +86,15 @@ public class ApplicationHandler extends HttpHandler {
         }
 
         if (isServiceConvergeListRequest(request)) {
-            return applicationRepository.serviceListToCheckForConfigConvergence(tenant, applicationId, request.getUri());
+            return applicationRepository.serviceListToCheckForConfigConvergence(applicationId, request.getUri());
         }
 
         if (isFiledistributionStatusRequest(request)) {
             Duration timeout = HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5));
-            return applicationRepository.filedistributionStatus(tenant, applicationId, timeout);
+            return applicationRepository.filedistributionStatus(applicationId, timeout);
         }
 
-        return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(tenant, applicationId));
+        return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId));
     }
 
     @Override
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
index 2c46f2968ce..a08b077699c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java
@@ -3,8 +3,10 @@ package com.yahoo.vespa.config.server.maintenance;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.model.api.FileDistribution;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.config.server.session.FileDistributionFactory;
 import com.yahoo.vespa.curator.Curator;
 
 import java.time.Duration;
@@ -13,20 +15,24 @@ public class ConfigServerMaintenance extends AbstractComponent {
 
     private final TenantsMaintainer tenantsMaintainer;
     private final ZooKeeperDataMaintainer zooKeeperDataMaintainer;
+    private final FileDistributionMaintainer fileDistributionMaintainer;
 
     @SuppressWarnings("unused") // instantiated by Dependency Injection
     public ConfigServerMaintenance(ConfigserverConfig configserverConfig,
                                    ApplicationRepository applicationRepository,
-                                   Curator curator) {
+                                   Curator curator,
+                                   FileDistributionFactory fileDistributionFactory) {
         DefaultTimes defaults = new DefaultTimes(configserverConfig);
         tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval);
-        zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.zookeeperDataMaintainerInterval);
+        zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.defaultInterval);
+        fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig);
     }
 
     @Override
     public void deconstruct() {
         tenantsMaintainer.deconstruct();
         zooKeeperDataMaintainer.deconstruct();
+        fileDistributionMaintainer.deconstruct();
     }
 
     /*
@@ -37,7 +43,6 @@ public class ConfigServerMaintenance extends AbstractComponent {
 
         private final Duration defaultInterval;
         private final Duration tenantsMaintainerInterval;
-        private final Duration zookeeperDataMaintainerInterval;
 
         DefaultTimes(ConfigserverConfig configserverConfig) {
             boolean isCd = configserverConfig.system().equals(SystemName.cd.name());
@@ -45,7 +50,6 @@ public class ConfigServerMaintenance extends AbstractComponent {
             this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes());
             // TODO: Want job control or feature flag to control when to run this, for now use a very long interval unless in CD
             this.tenantsMaintainerInterval = isCd ? defaultInterval : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes());
-            this.zookeeperDataMaintainerInterval = defaultInterval;
         }
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
new file mode 100644
index 00000000000..58141a3a045
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java
@@ -0,0 +1,33 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.maintenance;
+
+import com.yahoo.cloud.config.ConfigserverConfig;
+import com.yahoo.config.model.api.FileDistribution;
+import com.yahoo.vespa.config.server.ApplicationRepository;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.defaults.Defaults;
+
+import java.io.File;
+import java.time.Duration;
+
+public class FileDistributionMaintainer extends Maintainer {
+
+    private final ApplicationRepository applicationRepository;
+    private final File fileReferencesDir;
+
+    public FileDistributionMaintainer(ApplicationRepository applicationRepository,
+                                      Curator curator,
+                                      Duration interval,
+                                      ConfigserverConfig configserverConfig) {
+        super(applicationRepository, curator, interval);
+        this.applicationRepository = applicationRepository;
+        this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()));;
+    }
+
+
+    @Override
+    protected void maintain() {
+        // TODO: Does not delete, for now just outputs what should be deleted
+        applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, false);
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java
index d3a74486d12..8394494adca 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java
@@ -1,6 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.session;
 
+import com.google.inject.Inject;
+import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.vespa.config.server.filedistribution.FileDistributionImpl;
 import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider;
 
@@ -14,8 +16,15 @@ import java.io.File;
 @SuppressWarnings("WeakerAccess")
 public class FileDistributionFactory {
 
+    private final ConfigserverConfig configserverConfig;
+
+    @Inject
+    public FileDistributionFactory(ConfigserverConfig configserverConfig) {
+        this.configserverConfig = configserverConfig;
+    }
+
     public FileDistributionProvider createProvider(File applicationPackage) {
-        return new FileDistributionProvider(applicationPackage, new FileDistributionImpl());
+        return new FileDistributionProvider(applicationPackage, new FileDistributionImpl(configserverConfig));
     }
 
 }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
index 058e39eea9b..13a44f3c047 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server;
 
+import com.google.common.io.Files;
 import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
@@ -10,10 +11,9 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.TenantName;
-import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream;
-import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest;
+import com.yahoo.io.IOUtils;
+import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.server.http.SessionHandlerTest;
-import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler;
 import com.yahoo.vespa.config.server.http.v2.PrepareResult;
 import com.yahoo.vespa.config.server.session.PrepareParams;
 import com.yahoo.vespa.config.server.tenant.Tenant;
@@ -24,11 +24,13 @@ import org.junit.Before;
 import org.junit.Test;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -48,18 +50,19 @@ public class ApplicationRepositoryTest {
 
     private Tenant tenant;
     private ApplicationRepository applicationRepository;
+    private TenantRepository tenantRepository;
     private TimeoutBudget timeoutBudget;
 
     @Before
     public void setup() {
         Curator curator = new MockCurator();
-        TenantRepository tenants = new TenantRepository(new TestComponentRegistry.Builder()
+        tenantRepository = new TenantRepository(new TestComponentRegistry.Builder()
                                                                 .curator(curator)
                                                                 .build());
-        tenants.addTenant(tenantName);
-        tenant = tenants.getTenant(tenantName);
+        tenantRepository.addTenant(tenantName);
+        tenant = tenantRepository.getTenant(tenantName);
         Provisioner provisioner = new SessionHandlerTest.MockProvisioner();
-        applicationRepository = new ApplicationRepository(tenants, provisioner, clock);
+        applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock);
         timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60));
     }
 
@@ -79,15 +82,15 @@ public class ApplicationRepositoryTest {
     }
 
     @Test
-    public void createAndPrepareAndActivate() throws IOException {
-        PrepareResult result = deployApp();
+    public void createAndPrepareAndActivate() {
+        PrepareResult result = deployApp(testApp);
         assertTrue(result.configChangeActions().getRefeedActions().isEmpty());
         assertTrue(result.configChangeActions().getRestartActions().isEmpty());
     }
 
     @Test
-    public void deleteUnusedTenants() throws IOException {
-        deployApp();
+    public void deleteUnusedTenants() {
+        deployApp(testApp);
         assertTrue(applicationRepository.removeUnusedTenants().isEmpty());
         applicationRepository.remove(applicationId());
         assertEquals(tenantName, applicationRepository.removeUnusedTenants().iterator().next());
@@ -110,17 +113,48 @@ public class ApplicationRepositoryTest {
         assertEquals(Vtag.currentVersion, ApplicationRepository.decideVersion(regularApp, Environment.perf, targetVersion));
     }
 
+    @Test
+    public void deleteUnusedFileReferences() {
+        File fileReferencesDir = Files.createTempDir();
+
+        // Add file reference that is not in use and should be deleted
+        File filereferenceDir = new File(fileReferencesDir, "foo");
+        assertTrue(filereferenceDir.mkdir());
+        IOUtils.writeFile(new File(filereferenceDir, "bar"), Utf8.toBytes("test"));
+
+        tenantRepository.addTenant(tenantName);
+        tenant = tenantRepository.getTenant(tenantName);
+        Provisioner provisioner = new SessionHandlerTest.MockProvisioner();
+        applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock);
+        timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60));
+
+        // TODO: Deploy an app with a bundle or file that will be a file reference, too much missing in test setup to get this working now
+        PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build();
+        deployApp(new File("src/test/apps/app"), prepareParams);
+
+        boolean deleteFiles = false;
+        Set<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles);
+        assertEquals(new HashSet<>(Collections.singletonList("foo")), toBeDeleted);
+        assertTrue(filereferenceDir.exists());
+
+        deleteFiles = true;
+        toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles);
+        assertEquals(new HashSet<>(Collections.singletonList("foo")), toBeDeleted);
+        assertFalse(filereferenceDir.exists());
+    }
+
     private PrepareResult prepareAndActivateApp(File application) throws IOException {
         FilesApplicationPackage appDir = FilesApplicationPackage.fromFile(application);
         long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, appDir.getAppDir());
         return applicationRepository.prepareAndActivate(tenant, sessionId, prepareParams(), false, false, Instant.now());
     }
 
-    private PrepareResult deployApp() throws IOException {
-        File file = CompressedApplicationInputStreamTest.createTarFile();
-        return applicationRepository.deploy(CompressedApplicationInputStream.createFromCompressedStream(
-                                                    new FileInputStream(file), ApplicationApiHandler.APPLICATION_X_GZIP),
-                                            prepareParams(), false, false, Instant.now());
+    private PrepareResult deployApp(File applicationPackage) {
+        return deployApp(applicationPackage, prepareParams());
+    }
+
+    private PrepareResult deployApp(File applicationPackage, PrepareParams prepareParams) {
+        return applicationRepository.deploy(applicationPackage, prepareParams);
     }
 
     private PrepareParams prepareParams() {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java
index de27a185aef..099cf1e1d56 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java
@@ -60,7 +60,7 @@ public class InjectedGlobalComponentRegistryTest {
                         .configDefinitionsDir(Files.createTempDir().getAbsolutePath()));
         sessionPreparer = new SessionTest.MockSessionPreparer();
         rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), 
-                                  new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath()));
+                                  new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(Files.createTempDir()));
         generationCounter = new SuperModelGenerationCounter(curator);
         defRepo = new StaticConfigDefinitionRepo();
         permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig);
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java
index feb3ddae4ca..e4e45d3a014 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java
@@ -23,6 +23,7 @@ import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
 import com.yahoo.vespa.model.VespaModelFactory;
 
+import java.io.File;
 import java.time.Clock;
 import java.util.Collections;
 import java.util.Optional;
@@ -152,7 +153,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry {
             final PermanentApplicationPackage permApp = this.permanentApplicationPackage
                     .orElse(new PermanentApplicationPackage(configserverConfig));
             FileDistributionFactory fileDistributionFactory = this.fileDistributionFactory
-                    .orElse(new MockFileDistributionFactory());
+                    .orElse(new MockFileDistributionFactory(new File(configserverConfig.fileReferencesDir())));
             HostProvisionerProvider hostProvisionerProvider = hostProvisioner.isPresent() ?
                     HostProvisionerProvider.withProvisioner(hostProvisioner.get()) :
                     HostProvisionerProvider.empty();
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
index 157c36d7aef..1c0102e0b70 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.filedistribution;
 
+import com.google.common.io.Files;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.io.IOUtils;
 import com.yahoo.net.HostName;
@@ -73,8 +74,10 @@ public class FileServerTest {
     @Test
     public void requireThatDifferentNumberOfConfigServersWork() {
         // Empty connection pool in tests etc.
-        ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder();
-        FileServer fileServer = new FileServer(new ConfigserverConfig(builder));
+        ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder()
+                .configServerDBDir(Files.createTempDir().getAbsolutePath())
+                .configDefinitionsDir(Files.createTempDir().getAbsolutePath());
+        FileServer fileServer = createFileServer(builder);
         assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize());
 
         // Empty connection pool when only one server, no use in downloading from yourself
@@ -84,7 +87,7 @@ public class FileServerTest {
         serverBuilder.port(123456);
         servers.add(serverBuilder);
         builder.zookeeperserver(servers);
-        fileServer = new FileServer(new ConfigserverConfig(builder));
+        fileServer = createFileServer(builder);
         assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize());
 
         // connection pool of size 1 when 2 servers
@@ -93,10 +96,16 @@ public class FileServerTest {
         serverBuilder2.port(123456);
         servers.add(serverBuilder2);
         builder.zookeeperserver(servers);
-        fileServer = new FileServer(new ConfigserverConfig(builder));
+        fileServer = createFileServer(builder);
         assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize());
     }
 
+    private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) {
+        File fileReferencesDir = Files.createTempDir();
+        configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath());
+        return new FileServer(new ConfigserverConfig(configBuilder));
+    }
+
     private static class FileReceiver implements FileServer.Receiver {
         CompletableFuture<byte []> content;
         FileReceiver(CompletableFuture<byte []> content) {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java
index 5a9735f774a..f06f6c6e151 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java
@@ -1,8 +1,8 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.rpc;
 
+import com.google.common.io.Files;
 import com.yahoo.cloud.config.ConfigserverConfig;
-import com.yahoo.config.model.api.FileDistribution;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Version;
 import com.yahoo.vespa.config.protocol.ConfigResponse;
@@ -38,7 +38,7 @@ public class MockRpc extends RpcServer {
 
     public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication) {
         super(createConfig(port), null, Metrics.createTestMetrics(), 
-              new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath()));
+              new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(Files.createTempDir()));
         if (createDefaultTenant) {
             onTenantCreate(TenantName.from("default"), new MockTenantProvider(pretendToHaveLoadedAnyApplication));
         }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java
index e022b622fb0..eadf17a019d 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.rpc;
 
+import com.google.common.io.Files;
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.config.model.api.FileDistribution;
 import com.yahoo.config.provision.HostLivenessTracker;
@@ -94,7 +95,7 @@ public class TestWithRpc {
                                                                        emptyNodeFlavors(),
                                                                        generationCounter)),
                                   Metrics.createTestMetrics(), new HostRegistries(),
-                                  hostLivenessTracker, new FileServer(FileDistribution.getDefaultFileDBPath()));
+                                  hostLivenessTracker, new FileServer(Files.createTempDir()));
         rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider);
         t = new Thread(rpcServer);
         t.start();
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java
index 9d8b7c5cc00..2c05017d449 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java
@@ -1,6 +1,7 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.config.server.session;
 
+import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider;
 import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionProvider;
 
@@ -11,7 +12,12 @@ import java.io.File;
 */
 public class MockFileDistributionFactory extends FileDistributionFactory {
 
-    public final MockFileDistributionProvider mockFileDistributionProvider = new MockFileDistributionProvider();
+    public final MockFileDistributionProvider mockFileDistributionProvider;
+
+    public MockFileDistributionFactory(File fileReferencesDir) {
+        super(new ConfigserverConfig(new ConfigserverConfig.Builder()));
+        mockFileDistributionProvider = new MockFileDistributionProvider(fileReferencesDir);
+    }
 
     @Override
     public FileDistributionProvider createProvider(File applicationFile) {