diff options
3 files changed, 23 insertions, 20 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 818e65b6caf..37e550734c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -578,16 +578,18 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return fileDistributionStatus.status(getApplication(applicationId), timeout); } - public List<String> deleteUnusedFiledistributionReferences(File fileReferencesPath, Duration keepFileReferences) { - log.log(Level.FINE, () -> "Keep unused file references for " + keepFileReferences); + public List<String> deleteUnusedFiledistributionReferences(File fileReferencesPath, + Duration keepFileReferencesDuration, + int numberToAlwaysKeep) { + log.log(Level.FINE, () -> "Keep unused file references for " + keepFileReferencesDuration); if (!fileReferencesPath.isDirectory()) throw new RuntimeException(fileReferencesPath + " is not a directory"); Set<String> fileReferencesInUse = getFileReferencesInUse(); log.log(Level.FINE, () -> "File references in use : " + fileReferencesInUse); - List<String> candidates = sortedUnusedFileReferences(fileReferencesPath, fileReferencesInUse, keepFileReferences); + List<String> candidates = sortedUnusedFileReferences(fileReferencesPath, fileReferencesInUse, keepFileReferencesDuration); // Do not delete the newest ones - List<String> fileReferencesToDelete = candidates.subList(0, Math.max(0, candidates.size() - 5)); + List<String> fileReferencesToDelete = candidates.subList(0, Math.max(0, candidates.size() - numberToAlwaysKeep)); if (fileReferencesToDelete.size() > 0) { log.log(Level.FINE, () -> "Will delete file references not in use: " + fileReferencesToDelete); fileReferencesToDelete.forEach(fileReference -> { @@ -601,18 +603,12 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private Set<String> getFileReferencesInUse() { Set<String> fileReferencesInUse = new HashSet<>(); - // Intentionally skip applications that we for some reason do not find - // or that we fail to get file references for (they will be retried on the next run) for (var applicationId : listApplications()) { - try { - Optional<Application> app = getOptionalApplication(applicationId); - if (app.isEmpty()) continue; - fileReferencesInUse.addAll(app.get().getModel().fileReferences().stream() - .map(FileReference::value) - .collect(Collectors.toSet())); - } catch (Exception e) { - log.log(Level.WARNING, "Getting file references in use for '" + applicationId + "' failed", e); - } + Optional<Application> app = getOptionalApplication(applicationId); + if (app.isEmpty()) continue; + fileReferencesInUse.addAll(app.get().getModel().fileReferences().stream() + .map(FileReference::value) + .collect(Collectors.toSet())); } return fileReferencesInUse; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index ca8db30c21f..f0c1afdc6ac 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.maintenance; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.defaults.Defaults; @@ -10,7 +11,8 @@ import java.io.File; import java.time.Duration; /** - * Removes unused file references from disk + * Removes unused file references older than a configured time, but always keeps a certain number of file references + * even when they are unused. * <p> * Note: Unit test is in ApplicationRepositoryTest * @@ -18,6 +20,8 @@ import java.time.Duration; */ public class FileDistributionMaintainer extends ConfigServerMaintainer { + private static final int numberToAlwaysKeep = 10; + private final ApplicationRepository applicationRepository; private final File fileReferencesDir; private final Duration maxUnusedFileReferenceAge; @@ -28,13 +32,14 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer { FlagSource flagSource) { super(applicationRepository, curator, flagSource, applicationRepository.clock().instant(), interval); this.applicationRepository = applicationRepository; - this.maxUnusedFileReferenceAge = Duration.ofMinutes(applicationRepository.configserverConfig().keepUnusedFileReferencesMinutes()); - this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(applicationRepository.configserverConfig().fileReferencesDir())); + ConfigserverConfig configserverConfig = applicationRepository.configserverConfig(); + this.maxUnusedFileReferenceAge = Duration.ofMinutes(configserverConfig.keepUnusedFileReferencesMinutes()); + this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); } @Override protected double maintain() { - applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge); + applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge, numberToAlwaysKeep); return 1.0; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index e8dc08d4e8d..934440f03d1 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -300,7 +300,9 @@ public class ApplicationRepositoryTest { PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build(); deployApp(new File("src/test/apps/app"), prepareParams); - List<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, keepFileReferences); + List<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, + keepFileReferences, + 5); Collections.sort(toBeDeleted); assertEquals(List.of("bar0", "foo"), toBeDeleted); // bar0 and foo are the only ones that will be deleted (keeps 5 newest no matter how old they are) |