diff options
author | Harald Musum <musum@verizonmedia.com> | 2019-07-08 13:44:39 +0200 |
---|---|---|
committer | Harald Musum <musum@verizonmedia.com> | 2019-07-08 13:44:39 +0200 |
commit | 15d8f2cca9c8c3eea0b6c373f0f9a8544859e88d (patch) | |
tree | 7a79cbd4596ad8abeda4e8e72aca44a123305fae /config-proxy/src/main | |
parent | a173ee16ceba9c9192b2405887905d7beed3e388 (diff) |
Track when file reference or download was last requested
Diffstat (limited to 'config-proxy/src/main')
3 files changed, 48 insertions, 22 deletions
diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index 33a8ed405a9..0d72a1f02b6 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -29,7 +29,7 @@ import java.util.stream.Collectors; * * @author hmusum */ -public class FileDistributionRpcServer { +class FileDistributionRpcServer { private final static Logger log = Logger.getLogger(FileDistributionRpcServer.class.getName()); @@ -38,13 +38,13 @@ public class FileDistributionRpcServer { private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(), new DaemonThreadFactory("Rpc executor")); - public FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) { + FileDistributionRpcServer(Supervisor supervisor, FileDownloader downloader) { this.supervisor = supervisor; this.downloader = downloader; declareFileDistributionMethods(); } - public void close() { + void close() { rpcDownloadExecutor.shutdownNow(); try { rpcDownloadExecutor.awaitTermination(10, TimeUnit.SECONDS); @@ -80,8 +80,6 @@ public class FileDistributionRpcServer { private static final int baseFileProviderErrorCode = baseErrorCode + 0x1000; private static final int fileReferenceDoesNotExists = baseFileProviderErrorCode; - private static final int fileReferenceRemoved = fileReferenceDoesNotExists + 1; - private static final int fileReferenceInternalError = fileReferenceRemoved + 1; private void getFile(Request req) { req.detach(); @@ -118,19 +116,16 @@ public class FileDistributionRpcServer { private void downloadFile(Request req) { FileReference fileReference = new FileReference(req.parameters().get(0).asString()); log.log(LogLevel.DEBUG, () -> "getFile() called for file reference '" + fileReference.value() + "'"); - Optional<File> pathToFile = downloader.getFile(fileReference); - try { - if (pathToFile.isPresent()) { - req.returnValues().add(new StringValue(pathToFile.get().getAbsolutePath())); - log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' available at " + pathToFile.get()); - } else { - log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); - req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); - } - } catch (Throwable e) { - log.log(LogLevel.WARNING, "File reference '" + fileReference.value() + "' got exception: " + e.getMessage()); - req.setError(fileReferenceInternalError, "File reference '" + fileReference.value() + "' removed"); + Optional<File> file = downloader.getFile(fileReference); + if (file.isPresent()) { + new RequestTracker().trackRequest(file.get().getParentFile()); + req.returnValues().add(new StringValue(file.get().getAbsolutePath())); + log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' available at " + file.get()); + } else { + log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); + req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); } + req.returnRequest(); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java new file mode 100644 index 00000000000..47f478ea4d7 --- /dev/null +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java @@ -0,0 +1,30 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.proxy.filedistribution; + +import com.yahoo.log.LogLevel; + +import java.io.File; +import java.time.Instant; +import java.util.logging.Logger; + +/** + * Set last modification time for a file reference or downloaded url, to be able + * to later clean up file references or urls not used for a long time. + * + * @author hmusum + */ +class RequestTracker { + + private final static Logger log = Logger.getLogger(RequestTracker.class.getName()); + + void trackRequest(File file) { + String absolutePath = file.getAbsolutePath(); + if ( ! file.exists()) + log.log(LogLevel.WARNING, "Could not find file '" + absolutePath + "'"); + + if ( ! file.setLastModified(Instant.now().toEpochMilli())) + log.log(LogLevel.WARNING, "Could not set last modified timestamp for '" + absolutePath + "'"); + } + +} diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index 9d89f1d10b2..cdf079631fe 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -42,7 +42,7 @@ class UrlDownloadRpcServer { private final static Logger log = Logger.getLogger(UrlDownloadRpcServer.class.getName()); private static final String CONTENTS_FILE_NAME = "contents"; - private static final String LAST_MODFIED_FILE_NAME = "lastmodified"; + private static final String LAST_MODIFIED_FILE_NAME = "lastmodified"; private final File downloadBaseDir; private final ExecutorService rpcDownloadExecutor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()), @@ -110,16 +110,17 @@ class UrlDownloadRpcServer { if (contentsPath.exists() && contentsPath.length() > 0) { writeLastModifiedTimestamp(downloadDir, connection.getLastModified()); + new RequestTracker().trackRequest(downloadDir); req.returnValues().add(new StringValue(contentsPath.getAbsolutePath())); log.log(LogLevel.DEBUG, () -> "URL '" + url + "' available at " + contentsPath); + log.log(LogLevel.INFO, String.format("Download of URL '%s' done in %.3f seconds", + url, (System.currentTimeMillis() -start) / 1000.0)); } else { log.log(LogLevel.ERROR, "Downloaded URL '" + url + "' not found, returning error"); req.setError(DOES_NOT_EXIST, "Downloaded '" + url + "' not found"); } } } - long end = System.currentTimeMillis(); - log.log(LogLevel.INFO, String.format("Download of URL '%s' done in %.3f seconds", url, (end-start) / 1000.0)); } private static String urlToDirName(String uri) { @@ -137,7 +138,7 @@ class UrlDownloadRpcServer { } private static long readLastModifiedTimestamp(File downloadDir) throws IOException { - File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME); + File lastModified = new File(downloadDir, LAST_MODIFIED_FILE_NAME); if (lastModified.exists() && lastModified.length() > 0) { try (BufferedReader br = new BufferedReader(new FileReader(lastModified))) { String timestamp = br.readLine(); @@ -148,7 +149,7 @@ class UrlDownloadRpcServer { } private static void writeLastModifiedTimestamp(File downloadDir, long timestamp) throws IOException { - File lastModified = new File(downloadDir, LAST_MODFIED_FILE_NAME); + File lastModified = new File(downloadDir, LAST_MODIFIED_FILE_NAME); try (BufferedWriter lastModifiedWriter = new BufferedWriter(new FileWriter(lastModified.getAbsolutePath()))) { lastModifiedWriter.write(Long.toString(timestamp)); } |