diff options
Diffstat (limited to 'config-proxy/src/main')
3 files changed, 111 insertions, 3 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java new file mode 100644 index 00000000000..eec045cdb0a --- /dev/null +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java @@ -0,0 +1,92 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.proxy.filedistribution; + +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.filedistribution.FileDownloader; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static java.nio.file.Files.readAttributes; + +/** + * Deletes cached file references and url downloads that have not been used for some time + * + * @author hmusum + */ +class CachedFilesMaintainer implements Runnable { + + private final static Logger log = Logger.getLogger(CachedFilesMaintainer.class.getName()); + + private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir; + private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory; + private static final Duration defaultDurationToKeepFiles = Duration.ofDays(30); + + private final File urlDownloadDir; + private final File fileReferencesDownloadDir; + private final Duration durationToKeepFiles; + + CachedFilesMaintainer() { + this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, defaultDurationToKeepFiles); + } + + CachedFilesMaintainer(File fileReferencesDownloadDir, File urlDownloadDir, Duration durationToKeepFiles) { + this.fileReferencesDownloadDir = fileReferencesDownloadDir; + this.urlDownloadDir = urlDownloadDir; + this.durationToKeepFiles = durationToKeepFiles; + } + + @Override + public void run() { + try { + deleteUnusedFiles(fileReferencesDownloadDir); + deleteUnusedFiles(urlDownloadDir); + } catch (Throwable t) { + log.log(Level.WARNING, "Deleting unused files failed. ", t); + } + } + + private void deleteUnusedFiles(File directory) { + Instant deleteNotUsedSinceInstant = Instant.now().minus(durationToKeepFiles); + Set<String> filesOnDisk = new HashSet<>(); + File[] files = directory.listFiles(); + if (files != null) + filesOnDisk.addAll(Arrays.stream(files).map(File::getName).collect(Collectors.toSet())); + log.log(LogLevel.DEBUG, "Files on disk (in " + directory + "): " + filesOnDisk); + + Set<String> filesToDelete = filesOnDisk + .stream() + .filter(fileReference -> isFileLastModifiedBefore(new File(directory, fileReference), deleteNotUsedSinceInstant)) + .collect(Collectors.toSet()); + if (filesToDelete.size() > 0) { + log.log(LogLevel.INFO, "Files that can be deleted in " + directory + " (not used since " + deleteNotUsedSinceInstant + "): " + filesToDelete); + filesToDelete.forEach(fileReference -> { + File file = new File(directory, fileReference); + if (!IOUtils.recursiveDeleteDir(file)) + log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath()); + }); + } + } + + private boolean isFileLastModifiedBefore(File fileReference, Instant instant) { + BasicFileAttributes fileAttributes; + try { + fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class); + return fileAttributes.lastModifiedTime().toInstant().isBefore(instant); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index 0b7de6ed562..2767d2c8027 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -1,11 +1,17 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy.filedistribution; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; +import java.time.Duration; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + /** * Keeps track of file distribution and url download rpc servers. * @@ -13,17 +19,28 @@ import com.yahoo.vespa.filedistribution.FileDownloader; */ public class FileDistributionAndUrlDownload { + private static final Duration delay = Duration.ofMinutes(1); private final FileDistributionRpcServer fileDistributionRpcServer; private final UrlDownloadRpcServer urlDownloadRpcServer; + private final ScheduledExecutorService cleanupExecutor = + new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup")); public FileDistributionAndUrlDownload(Supervisor supervisor, ConfigSourceSet source) { fileDistributionRpcServer = new FileDistributionRpcServer(supervisor, new FileDownloader(new JRTConnectionPool(source))); urlDownloadRpcServer = new UrlDownloadRpcServer(supervisor); + cleanupExecutor.scheduleAtFixedRate(new CachedFilesMaintainer(), delay.toSeconds(), delay.toSeconds(), TimeUnit.SECONDS); } public void close() { fileDistributionRpcServer.close(); urlDownloadRpcServer.close(); + cleanupExecutor.shutdownNow(); + try { + if ( ! cleanupExecutor.awaitTermination(10, TimeUnit.SECONDS)) + throw new RuntimeException("Unable to shutdown " + cleanupExecutor + " before timeout"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index cdf079631fe..592f5211eed 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -44,7 +44,7 @@ class UrlDownloadRpcServer { private static final String CONTENTS_FILE_NAME = "contents"; private static final String LAST_MODIFIED_FILE_NAME = "lastmodified"; - private final File downloadBaseDir; + static final File downloadDir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/download")); private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), new DaemonThreadFactory("Rpc URL download executor")); @@ -53,7 +53,6 @@ class UrlDownloadRpcServer { .methodDesc("get path to url download") .paramDesc(0, "url", "url") .returnDesc(0, "path", "path to file")); - downloadBaseDir = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/download")); } void close() { @@ -72,7 +71,7 @@ class UrlDownloadRpcServer { private void downloadFile(Request req) { String url = req.parameters().get(0).asString(); - File downloadDir = new File(this.downloadBaseDir, urlToDirName(url)); + File downloadDir = new File(UrlDownloadRpcServer.downloadDir, urlToDirName(url)); try { URL website = new URL(url); |