From 06074fd0835fc94bd93f6b2b530a954f0e8980b6 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 25 Jul 2023 10:49:11 +0200 Subject: Do not run maintainer on config server hosts Config server has its own maintainer, running this one might remove files that are still in use (belonging to an active application), so don't run it if this host is one in VESPA_CONFIGSERVERS or it's empty/null --- .../FileReferencesAndDownloadsMaintainer.java | 57 ++++++++++++++++------ .../FileReferencesAndDownloadsMaintainerTest.java | 18 ++++++- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java index 091f3eebc21..eab1368a2a1 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.proxy.filedistribution; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.filedistribution.FileDownloader; import java.io.File; @@ -15,6 +16,7 @@ import java.time.Instant; import java.util.Comparator; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -23,6 +25,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import static java.nio.file.Files.readAttributes; +import static java.util.logging.Level.INFO; /** * Deletes file references and url downloads that have not been used for some time. @@ -40,30 +43,40 @@ class FileReferencesAndDownloadsMaintainer implements Runnable { private static final int defaultOutdatedFilesToKeep = 20; private static final Duration interval = Duration.ofMinutes(1); - private final ScheduledExecutorService executor = - new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup")); + private final Optional executor; private final File urlDownloadDir; private final File fileReferencesDownloadDir; private final Duration durationToKeepFiles; private final int outDatedFilesToKeep; FileReferencesAndDownloadsMaintainer() { - this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, keepFileReferencesDuration(), outDatedFilesToKeep()); + this(defaultFileReferencesDownloadDir, defaultUrlDownloadDir, keepFileReferencesDuration(), + outDatedFilesToKeep(), configServers()); } FileReferencesAndDownloadsMaintainer(File fileReferencesDownloadDir, File urlDownloadDir, Duration durationToKeepFiles, - int outdatedFilesToKeep) { + int outdatedFilesToKeep, + List configServers) { this.fileReferencesDownloadDir = fileReferencesDownloadDir; this.urlDownloadDir = urlDownloadDir; this.durationToKeepFiles = durationToKeepFiles; this.outDatedFilesToKeep = outdatedFilesToKeep; - executor.scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); + // Do not run on config servers + if (configServers.contains(ConfigUtils.getCanonicalHostName())) { + log.log(INFO, "Not running maintainer, since this is on a config server host"); + executor = Optional.empty(); + } else { + executor = Optional.of(new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("file references and downloads cleanup"))); + executor.get().scheduleAtFixedRate(this, interval.toSeconds(), interval.toSeconds(), TimeUnit.SECONDS); + } } @Override public void run() { + if (executor.isEmpty()) return; + try { deleteUnusedFiles(fileReferencesDownloadDir); deleteUnusedFiles(urlDownloadDir); @@ -73,13 +86,15 @@ class FileReferencesAndDownloadsMaintainer implements Runnable { } public void close() { - executor.shutdownNow(); - try { - if ( ! executor.awaitTermination(10, TimeUnit.SECONDS)) - throw new RuntimeException("Unable to shutdown " + executor + " before timeout"); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } + executor.ifPresent(ex -> { + ex.shutdownNow(); + try { + if (! ex.awaitTermination(10, TimeUnit.SECONDS)) + throw new RuntimeException("Unable to shutdown " + executor + " before timeout"); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); } private void deleteUnusedFiles(File directory) { @@ -111,9 +126,10 @@ class FileReferencesAndDownloadsMaintainer implements Runnable { // Make sure we keep some files canBeDeleted = canBeDeleted.subList(0, Math.min(canBeDeleted.size(), deleteCount)); - log.log(Level.INFO, "Files that can be deleted (not accessed since " + deleteNotUsedSinceInstant + - ", will also keep " + outDatedFilesToKeep + - " no matter when last accessed): " + canBeDeleted); + if (canBeDeleted.size() > 0) + log.log(INFO, "Files that can be deleted (not accessed since " + deleteNotUsedSinceInstant + + ", will also keep " + outDatedFilesToKeep + + " no matter when last accessed): " + canBeDeleted); return canBeDeleted; } @@ -140,11 +156,20 @@ class FileReferencesAndDownloadsMaintainer implements Runnable { } private static int outDatedFilesToKeep() { - String env = System.getenv("VESPA_KEEP_OUTDATED_FILE_REFERENCES_COUNT"); + String env = System.getenv("VESPA_KEEP_FILE_REFERENCES_COUNT"); if (env != null && !env.isEmpty()) return Integer.parseInt(env); else return defaultOutdatedFilesToKeep; } + private static List configServers() { + String env = System.getenv("VESPA_CONFIGSERVERS"); + if (env == null || env.isEmpty()) + return List.of(ConfigUtils.getCanonicalHostName()); + else { + return List.of(env.split(",")); + } + } + } diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainerTest.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainerTest.java index 19e2d99c3ae..c41305b4dc8 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainerTest.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainerTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.config.proxy.filedistribution; import com.yahoo.io.IOUtils; +import com.yahoo.vespa.config.util.ConfigUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -36,11 +37,12 @@ public class FileReferencesAndDownloadsMaintainerTest { public void setup() throws IOException { cachedFileReferences = newFolder(tempFolder, "cachedFileReferences"); cachedDownloads = newFolder(tempFolder, "cachedDownloads"); - maintainer = new FileReferencesAndDownloadsMaintainer(cachedFileReferences, cachedDownloads, keepDuration, outDatedFilesToKeep); } @Test void require_old_files_to_be_deleted() { + maintainer = new FileReferencesAndDownloadsMaintainer(cachedFileReferences, cachedDownloads, keepDuration, outDatedFilesToKeep, + List.of("host1")); runMaintainerAndAssertFiles(0, 0); var fileReferences = writeFiles(20); @@ -55,6 +57,20 @@ public class FileReferencesAndDownloadsMaintainerTest { runMaintainerAndAssertFiles(outDatedFilesToKeep, outDatedFilesToKeep); } + @Test + void require_no_files_deleted_when_running_on_config_server_host() { + maintainer = new FileReferencesAndDownloadsMaintainer(cachedFileReferences, cachedDownloads, keepDuration, + outDatedFilesToKeep, List.of(ConfigUtils.getCanonicalHostName())); + runMaintainerAndAssertFiles(0, 0); + + var fileReferences = writeFiles(10); + var downloads = writeDownloads(10); + runMaintainerAndAssertFiles(10, 10); + + updateLastModifiedTimestamp(0, 10, fileReferences, downloads); + runMaintainerAndAssertFiles(10, 10); + } + private void updateLastModifiedTimestamp(int startInclusive, int endExclusive, List fileReferences, List downloads) { IntStream.range(startInclusive, endExclusive).forEach(i -> { Instant instant = Instant.now().minus(keepDuration.plus(Duration.ofMinutes(1)).minus(Duration.ofSeconds(i))); -- cgit v1.2.3