From 2baa943016a94f33e0a6ff7ad41af0bc3a2260c5 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 7 Oct 2022 14:59:22 +0200 Subject: Move code for deleting unused file references No functional changes, will be used by config proxy code as well in the future --- .../vespa/config/server/ApplicationRepository.java | 59 ++-------------------- .../filedistribution/FileDistributionUtil.java | 18 +------ .../maintenance/FileDistributionMaintainer.java | 4 +- .../config/server/ApplicationRepositoryTest.java | 8 +-- 4 files changed, 13 insertions(+), 76 deletions(-) (limited to 'configserver') diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 2a15f724b29..f832c504526 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.config.server; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; -import com.yahoo.component.annotation.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; @@ -83,20 +83,18 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.stats.LockStats; import com.yahoo.vespa.curator.stats.ThreadLockStats; import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.filedistribution.maintenance.FileDistributionCleanup; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; import java.nio.file.Files; -import java.nio.file.attribute.BasicFileAttributes; import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collection; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -116,11 +114,9 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVE import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; -import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk; import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT; import static com.yahoo.vespa.curator.Curator.CompletionWaiter; import static com.yahoo.yolean.Exceptions.uncheck; -import static java.nio.file.Files.readAttributes; /** * The API for managing applications. @@ -590,30 +586,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return fileDistributionStatus.status(getApplication(applicationId), timeout); } - public List deleteUnusedFileDistributionReferences(File fileReferencesPath, - Duration keepFileReferencesDuration, - int numberToAlwaysKeep) { - log.log(Level.FINE, () -> "Keep unused file references for " + keepFileReferencesDuration); - if (!fileReferencesPath.isDirectory()) throw new RuntimeException(fileReferencesPath + " is not a directory"); - - Set fileReferencesInUse = getFileReferencesInUse(); - log.log(Level.FINE, () -> "File references in use : " + fileReferencesInUse); - - List candidates = sortedUnusedFileReferences(fileReferencesPath, fileReferencesInUse, keepFileReferencesDuration); - // Do not delete the newest ones - List fileReferencesToDelete = candidates.subList(0, Math.max(0, candidates.size() - numberToAlwaysKeep)); - if (fileReferencesToDelete.size() > 0) { - log.log(Level.FINE, () -> "Will delete file references not in use: " + fileReferencesToDelete); - fileReferencesToDelete.forEach(fileReference -> { - File file = new File(fileReferencesPath, fileReference); - if ( ! IOUtils.recursiveDeleteDir(file)) - log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath()); - }); - } - return fileReferencesToDelete; + public void deleteUnusedFileDistributionReferences(File fileReferencesPath, Duration keepFileReferencesDuration) { + new FileDistributionCleanup(clock).deleteUnusedFileReferences(fileReferencesPath, keepFileReferencesDuration, getFileReferencesInUse()); } - private Set getFileReferencesInUse() { + Set getFileReferencesInUse() { Set fileReferencesInUse = new HashSet<>(); for (var applicationId : listApplications()) { Application app = getApplication(applicationId); @@ -624,18 +601,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return fileReferencesInUse; } - private List sortedUnusedFileReferences(File fileReferencesPath, Set fileReferencesInUse, Duration keepFileReferences) { - Set fileReferencesOnDisk = getFileReferencesOnDisk(fileReferencesPath); - log.log(Level.FINE, () -> "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); - Instant instant = clock.instant().minus(keepFileReferences); - return fileReferencesOnDisk - .stream() - .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) - .filter(fileReference -> isLastFileAccessBefore(new File(fileReferencesPath, fileReference), instant)) - .sorted(Comparator.comparing(a -> lastAccessed(new File(fileReferencesPath, a)))) - .collect(Collectors.toList()); - } - public Set getFileReferences(ApplicationId applicationId) { return getOptionalApplication(applicationId).map(app -> app.getModel().fileReferences()).orElse(Set.of()); } @@ -686,20 +651,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .collect(Collectors.toList()); } - private boolean isLastFileAccessBefore(File fileReference, Instant instant) { - return lastAccessed(fileReference).isBefore(instant); - } - - private Instant lastAccessed(File fileReference) { - BasicFileAttributes fileAttributes; - try { - fileAttributes = readAttributes(fileReference.toPath(), BasicFileAttributes.class); - return fileAttributes.lastAccessTime().toInstant(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - public Optional getApplicationPackageReference(ApplicationId applicationId) { Optional applicationPackage = Optional.empty(); Optional session = getActiveSession(applicationId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java index a1ddad7bfd4..1fd842d03e5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionUtil.java @@ -5,12 +5,9 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; import com.yahoo.net.HostName; import com.yahoo.vespa.config.server.ConfigServerSpec; - +import com.yahoo.vespa.filedistribution.maintenance.FileDistributionCleanup; import java.io.File; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; /** @@ -21,17 +18,6 @@ import java.util.stream.Collectors; */ public class FileDistributionUtil { - /** - * Returns all files in the given directory, non-recursive. - */ - public static Set getFileReferencesOnDisk(File directory) { - Set fileReferencesOnDisk = new HashSet<>(); - File[] filesOnDisk = directory.listFiles(); - if (filesOnDisk != null) - fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); - return fileReferencesOnDisk; - } - public static List getOtherConfigServersInCluster(ConfigserverConfig configserverConfig) { return ConfigServerSpec.fromConfig(configserverConfig) .stream() @@ -41,7 +27,7 @@ public class FileDistributionUtil { } public static boolean fileReferenceExistsOnDisk(File downloadDirectory, FileReference applicationPackageReference) { - return getFileReferencesOnDisk(downloadDirectory).contains(applicationPackageReference.value()); + return FileDistributionCleanup.getFileReferencesOnDisk(downloadDirectory).contains(applicationPackageReference.value()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java index f6aee416c9c..5542d24253b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -20,8 +20,6 @@ import java.time.Duration; */ public class FileDistributionMaintainer extends ConfigServerMaintainer { - private static final int numberToAlwaysKeep = 20; - private final ApplicationRepository applicationRepository; private final File fileReferencesDir; private final Duration maxUnusedFileReferenceAge; @@ -39,7 +37,7 @@ public class FileDistributionMaintainer extends ConfigServerMaintainer { @Override protected double maintain() { - applicationRepository.deleteUnusedFileDistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge, numberToAlwaysKeep); + applicationRepository.deleteUnusedFileDistributionReferences(fileReferencesDir, maxUnusedFileReferenceAge); return 1.0; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 99487230c5d..4f7be104b9c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -49,6 +49,7 @@ import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.config.util.ConfigUtils; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.filedistribution.maintenance.FileDistributionCleanup; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.model.VespaModelFactory; import org.junit.Before; @@ -293,9 +294,10 @@ public class ApplicationRepositoryTest { PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build(); deployApp(new File("src/test/apps/app"), prepareParams); - List toBeDeleted = applicationRepository.deleteUnusedFileDistributionReferences(fileReferencesDir, - keepFileReferencesDuration, - 2); + List toBeDeleted = new FileDistributionCleanup(clock).deleteUnusedFileReferences(fileReferencesDir, + keepFileReferencesDuration, + 2, + applicationRepository.getFileReferencesInUse()); Collections.sort(toBeDeleted); assertEquals(List.of("bar0", "foo"), toBeDeleted); // bar0 and foo are the only ones that will be deleted (keeps 2 newest no matter how old they are) -- cgit v1.2.3