diff options
author | Harald Musum <musum@oath.com> | 2018-05-30 11:27:58 +0200 |
---|---|---|
committer | Harald Musum <musum@oath.com> | 2018-05-30 11:27:58 +0200 |
commit | 58a8bf83a3234f51ebab26a641ceec1bd01023e0 (patch) | |
tree | 1b419fe4cf15fb6de326c80b5e527854384d525d | |
parent | 8c2db62342f378c70db71406b535caf7d38a916e (diff) |
Add maintainer for deleting unused file references on disk
* For now the maintainer just outputs what to be deleted
* Add config for file references dir insteaed of hardcoding
to make it possible to test
21 files changed, 237 insertions, 66 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java index ac1bcfa542a..9b457f49bd2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/FileDistribution.java @@ -23,12 +23,18 @@ public interface FileDistribution { */ void startDownload(String hostName, int port, Set<FileReference> fileReferences); + // TODO: Remove when 6.244 is oldest version in use + @Deprecated static String getDefaultFileDBRoot() { return Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"); } + // TODO: Remove when 6.244 is oldest version in use + @Deprecated static File getDefaultFileDBPath() { return new File(getDefaultFileDBRoot()); } + File getFileReferencesDir(); + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java index eabd0e5a7e0..131a5344116 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/FileDistributorTestCase.java @@ -7,10 +7,10 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.test.MockHosts; import org.junit.Test; +import java.io.File; import java.util.Arrays; import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -48,5 +48,10 @@ public class FileDistributorTestCase { public void startDownload(String hostName, int port, Set<FileReference> fileReferences) { filesToDownloadCalled++; } + + @Override + public File getFileReferencesDir() { + return null; + } } } diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 5e81526dc53..bf4c9599f4a 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -13,10 +13,13 @@ zookeeperserver[].port int default=2181 zookeeper.barrierTimeout long default=120 # in seconds zookeeperLocalhostAffinity bool default=true -# Misc +# Directories configModelPluginDir[] string configServerDBDir string default="var/db/vespa/config_server/serverdb/" configDefinitionsDir string default="share/vespa/configdefinitions/" +fileReferencesDir string default="var/db/vespa/filedistribution/" + +# Misc sessionLifetime long default=3600 # in seconds masterGeneration long default=0 multitenant bool default=false diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 102a36e833f..07809feb057 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -7,6 +7,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.DeployLogger; @@ -54,7 +55,9 @@ import java.net.URI; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -269,29 +272,61 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return true; } - public HttpResponse clusterControllerStatusPage(Tenant tenant, ApplicationId applicationId, String hostName, String pathSuffix) { - Application application = getApplication(tenant, applicationId); - + public HttpResponse clusterControllerStatusPage(ApplicationId applicationId, String hostName, String pathSuffix) { // WARNING: pathSuffix may be given by the external user. Make sure no security issues arise... // We should be OK here, because at most, pathSuffix may change the parent path, but cannot otherwise // change the hostname and port. Exposing other paths on the cluster controller should be fine. // TODO: It would be nice to have a simple check to verify pathSuffix doesn't contain /../ components. String relativePath = "clustercontroller-status/" + pathSuffix; - return httpProxy.get(application, hostName, "container-clustercontroller", relativePath); + return httpProxy.get(getApplication(applicationId), hostName, "container-clustercontroller", relativePath); } - public Long getApplicationGeneration(Tenant tenant, ApplicationId applicationId) { - return getApplication(tenant, applicationId).getApplicationGeneration(); + public Long getApplicationGeneration(ApplicationId applicationId) { + return getApplication(applicationId).getApplicationGeneration(); } public void restart(ApplicationId applicationId, HostFilter hostFilter) { hostProvisioner.ifPresent(provisioner -> provisioner.restart(applicationId, hostFilter)); } - public HttpResponse filedistributionStatus(Tenant tenant, ApplicationId applicationId, Duration timeout) { - Application application = getApplication(tenant, applicationId); - return fileDistributionStatus.status(application, timeout); + public HttpResponse filedistributionStatus(ApplicationId applicationId, Duration timeout) { + return fileDistributionStatus.status(getApplication(applicationId), timeout); + } + + public Set<String> deleteUnusedFiledistributionReferences(File fileReferencesPath, boolean deleteFromDisk) { + // Find all file references in use + Set<String> fileReferencesInUse = new HashSet<>(); + Set<ApplicationId> applicationIds = listApplications(); + applicationIds.forEach(applicationId -> fileReferencesInUse.addAll(getApplication(applicationId).getModel().fileReferences() + .stream() + .map(FileReference::value) + .collect(Collectors.toSet()))); + log.log(LogLevel.INFO, "File references in use : " + fileReferencesInUse); + + // Find those on disk that are not in use + if (!fileReferencesPath.isDirectory()) + throw new RuntimeException(fileReferencesPath + " is not a directory"); + Set<String> fileReferencesOnDisk = new HashSet<>(); + File[] filesOnDisk = fileReferencesPath.listFiles(); + if (filesOnDisk != null) + fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); + log.log(LogLevel.INFO, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + + // TODO: Only consider the ones modified more than some time (14 days?) ago + Set<String> fileReferencesToDelete = fileReferencesOnDisk + .stream() + .filter(fileReference -> ! fileReferencesInUse.contains(fileReference)) + .collect(Collectors.toSet()); + if (deleteFromDisk) { + log.log(LogLevel.INFO, "Will delete file references not in use: " + fileReferencesToDelete); + fileReferencesToDelete.forEach(fileReference -> { + File file = new File(fileReferencesPath, fileReference); + if ( ! IOUtils.recursiveDeleteDir(file)) + log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath()); + }); + } + return fileReferencesToDelete; } public ApplicationFile getApplicationFileFromSession(TenantName tenantName, long sessionId, String path, LocalSession.Mode mode) { @@ -299,22 +334,27 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return getLocalSession(tenant, sessionId).getApplicationFile(Path.fromString(path), mode); } - private Application getApplication(Tenant tenant, ApplicationId applicationId) { + private Application getApplication(ApplicationId applicationId) { + Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); long sessionId = getSessionIdForApplication(tenant, applicationId); RemoteSession session = tenant.getRemoteSessionRepo().getSession(sessionId, 0); return session.ensureApplicationLoaded().getForVersionOrLatest(Optional.empty(), clock.instant()); } + private Set<ApplicationId> listApplications() { + return tenantRepository.getAllTenants().stream() + .flatMap(tenant -> tenant.getApplicationRepo().listApplications().stream()) + .collect(Collectors.toSet()); + } + // ---------------- Convergence ---------------------------------------------------------------- - public HttpResponse serviceConvergenceCheck(Tenant tenant, ApplicationId applicationId, String hostname, URI uri) { - Application application = getApplication(tenant, applicationId); - return convergeChecker.serviceConvergenceCheck(application, hostname, uri); + public HttpResponse serviceConvergenceCheck(ApplicationId applicationId, String hostname, URI uri) { + return convergeChecker.serviceConvergenceCheck(getApplication(applicationId), hostname, uri); } - public HttpResponse serviceListToCheckForConfigConvergence(Tenant tenant, ApplicationId applicationId, URI uri) { - Application application = getApplication(tenant, applicationId); - return convergeChecker.serviceListToCheckForConfigConvergence(application, uri); + public HttpResponse serviceListToCheckForConfigConvergence(ApplicationId applicationId, URI uri) { + return convergeChecker.serviceListToCheckForConfigConvergence(getApplication(applicationId), uri); } // ---------------- Session operations ---------------------------------------------------------------- diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 1c2c24cc7bb..df2287c64cb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -26,10 +26,6 @@ public class FileDirectory { private static final Logger log = Logger.getLogger(FileDirectory.class.getName()); private final File root; - public FileDirectory() { - this(FileDistribution.getDefaultFileDBPath()); - } - public FileDirectory(File rootDir) { root = rootDir; try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java index 544451b8e10..2db89c2e8ed 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.jrt.ErrorCode; @@ -11,7 +12,9 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.defaults.Defaults; +import java.io.File; import java.util.Set; import java.util.logging.Logger; @@ -22,12 +25,22 @@ public class FileDistributionImpl implements FileDistribution { private final static Logger log = Logger.getLogger(FileDistributionImpl.class.getName()); private final Supervisor supervisor = new Supervisor(new Transport()); + private final File fileReferencesDir; + + public FileDistributionImpl(ConfigserverConfig configserverConfig) { + this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())); + } @Override public void startDownload(String hostName, int port, Set<FileReference> fileReferences) { startDownloadingFileReferences(hostName, port, fileReferences); } + @Override + public File getFileReferencesDir() { + return fileReferencesDir; + } + // Notifies config proxy which file references it should start downloading. It's OK if the call does not succeed, // as downloading will then start synchronously when a service requests a file reference instead private void startDownloadingFileReferences(String hostName, int port, Set<FileReference> fileReferences) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java index d6751987424..b3f3214793c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java @@ -17,8 +17,8 @@ public class FileDistributionProvider { private final FileDistribution fileDistribution; public FileDistributionProvider(File applicationDir, FileDistribution fileDistribution) { - this(new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory())), fileDistribution); - ensureDirExists(FileDistribution.getDefaultFileDBPath()); + this(new FileDBRegistry(new ApplicationFileManager(applicationDir, new FileDirectory(fileDistribution.getFileReferencesDir()))), fileDistribution); + ensureDirExists(fileDistribution.getFileReferencesDir()); } FileDistributionProvider(FileRegistry fileRegistry, FileDistribution fileDistribution) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 001ef751e69..42bf269e9d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.config.server.filedistribution; import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.FileReference; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; @@ -17,6 +16,7 @@ import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.config.server.ConfigServerSpec; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.filedistribution.CompressedFileReference; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceData; @@ -72,7 +72,7 @@ public class FileServer { @Inject public FileServer(ConfigserverConfig configserverConfig) { - this(createConnectionPool(configserverConfig), FileDistribution.getDefaultFileDBPath()); + this(createConnectionPool(configserverConfig), new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()))); } // For testing only diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java index 728f327c829..40d75d9dbac 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDBHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistribution.java @@ -4,14 +4,23 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; +import java.io.File; import java.util.Set; /** * @author Ulf Lilleengen */ -public class MockFileDBHandler implements FileDistribution { +public class MockFileDistribution implements FileDistribution { + private final File fileReferencesDir; + + MockFileDistribution(File fileReferencesDir) { + this.fileReferencesDir = fileReferencesDir; + } @Override public void startDownload(String hostName, int port, Set<FileReference> fileReferences) {} + @Override + public File getFileReferencesDir() { return fileReferencesDir; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java index b4ed2352d00..db70a51b2b4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/MockFileDistributionProvider.java @@ -4,14 +4,16 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.model.application.provider.MockFileRegistry; +import java.io.File; + /** * @author Ulf Lilleengen */ public class MockFileDistributionProvider extends FileDistributionProvider { public int timesCalled = 0; - public MockFileDistributionProvider() { - super(new MockFileRegistry(), new MockFileDBHandler()); + public MockFileDistributionProvider(File fileReferencesDir) { + super(new MockFileRegistry(), new MockFileDistribution(fileReferencesDir)); } public FileDistribution getFileDistribution() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 42fdb16c7ca..6bca8b1c562 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -59,13 +59,13 @@ public class ApplicationHandler extends HttpHandler { Tenant tenant = verifyTenantAndApplication(applicationId); if (isServiceConvergeRequest(request)) { - return applicationRepository.serviceConvergenceCheck(tenant, applicationId, getHostNameFromRequest(request), request.getUri()); + return applicationRepository.serviceConvergenceCheck(applicationId, getHostNameFromRequest(request), request.getUri()); } if (isClusterControllerStatusRequest(request)) { String hostName = getHostNameFromRequest(request); String pathSuffix = getPathSuffix(request); - return applicationRepository.clusterControllerStatusPage(tenant, applicationId, hostName, pathSuffix); + return applicationRepository.clusterControllerStatusPage(applicationId, hostName, pathSuffix); } if (isContentRequest(request)) { @@ -86,15 +86,15 @@ public class ApplicationHandler extends HttpHandler { } if (isServiceConvergeListRequest(request)) { - return applicationRepository.serviceListToCheckForConfigConvergence(tenant, applicationId, request.getUri()); + return applicationRepository.serviceListToCheckForConfigConvergence(applicationId, request.getUri()); } if (isFiledistributionStatusRequest(request)) { Duration timeout = HttpHandler.getRequestTimeout(request, Duration.ofSeconds(5)); - return applicationRepository.filedistributionStatus(tenant, applicationId, timeout); + return applicationRepository.filedistributionStatus(applicationId, timeout); } - return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(tenant, applicationId)); + return new GetApplicationResponse(Response.Status.OK, applicationRepository.getApplicationGeneration(applicationId)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 2c46f2968ce..a08b077699c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -3,8 +3,10 @@ package com.yahoo.vespa.config.server.maintenance; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.AbstractComponent; +import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.config.server.session.FileDistributionFactory; import com.yahoo.vespa.curator.Curator; import java.time.Duration; @@ -13,20 +15,24 @@ public class ConfigServerMaintenance extends AbstractComponent { private final TenantsMaintainer tenantsMaintainer; private final ZooKeeperDataMaintainer zooKeeperDataMaintainer; + private final FileDistributionMaintainer fileDistributionMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ConfigServerMaintenance(ConfigserverConfig configserverConfig, ApplicationRepository applicationRepository, - Curator curator) { + Curator curator, + FileDistributionFactory fileDistributionFactory) { DefaultTimes defaults = new DefaultTimes(configserverConfig); tenantsMaintainer = new TenantsMaintainer(applicationRepository, curator, defaults.tenantsMaintainerInterval); - zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.zookeeperDataMaintainerInterval); + zooKeeperDataMaintainer = new ZooKeeperDataMaintainer(applicationRepository, curator, defaults.defaultInterval); + fileDistributionMaintainer = new FileDistributionMaintainer(applicationRepository, curator, defaults.defaultInterval, configserverConfig); } @Override public void deconstruct() { tenantsMaintainer.deconstruct(); zooKeeperDataMaintainer.deconstruct(); + fileDistributionMaintainer.deconstruct(); } /* @@ -37,7 +43,6 @@ public class ConfigServerMaintenance extends AbstractComponent { private final Duration defaultInterval; private final Duration tenantsMaintainerInterval; - private final Duration zookeeperDataMaintainerInterval; DefaultTimes(ConfigserverConfig configserverConfig) { boolean isCd = configserverConfig.system().equals(SystemName.cd.name()); @@ -45,7 +50,6 @@ public class ConfigServerMaintenance extends AbstractComponent { this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); // TODO: Want job control or feature flag to control when to run this, for now use a very long interval unless in CD this.tenantsMaintainerInterval = isCd ? defaultInterval : Duration.ofMinutes(configserverConfig.tenantsMaintainerIntervalMinutes()); - this.zookeeperDataMaintainerInterval = defaultInterval; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java new file mode 100644 index 00000000000..58141a3a045 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/FileDistributionMaintainer.java @@ -0,0 +1,33 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.maintenance; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.vespa.config.server.ApplicationRepository; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.defaults.Defaults; + +import java.io.File; +import java.time.Duration; + +public class FileDistributionMaintainer extends Maintainer { + + private final ApplicationRepository applicationRepository; + private final File fileReferencesDir; + + public FileDistributionMaintainer(ApplicationRepository applicationRepository, + Curator curator, + Duration interval, + ConfigserverConfig configserverConfig) { + super(applicationRepository, curator, interval); + this.applicationRepository = applicationRepository; + this.fileReferencesDir = new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir()));; + } + + + @Override + protected void maintain() { + // TODO: Does not delete, for now just outputs what should be deleted + applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, false); + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java index d3a74486d12..8394494adca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/FileDistributionFactory.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.google.inject.Inject; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.vespa.config.server.filedistribution.FileDistributionImpl; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; @@ -14,8 +16,15 @@ import java.io.File; @SuppressWarnings("WeakerAccess") public class FileDistributionFactory { + private final ConfigserverConfig configserverConfig; + + @Inject + public FileDistributionFactory(ConfigserverConfig configserverConfig) { + this.configserverConfig = configserverConfig; + } + public FileDistributionProvider createProvider(File applicationPackage) { - return new FileDistributionProvider(applicationPackage, new FileDistributionImpl()); + return new FileDistributionProvider(applicationPackage, new FileDistributionImpl(configserverConfig)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 058e39eea9b..13a44f3c047 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; +import com.google.common.io.Files; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.model.application.provider.FilesApplicationPackage; @@ -10,10 +11,9 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStream; -import com.yahoo.vespa.config.server.http.CompressedApplicationInputStreamTest; +import com.yahoo.io.IOUtils; +import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; import com.yahoo.vespa.config.server.http.v2.PrepareResult; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -24,11 +24,13 @@ import org.junit.Before; import org.junit.Test; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -48,18 +50,19 @@ public class ApplicationRepositoryTest { private Tenant tenant; private ApplicationRepository applicationRepository; + private TenantRepository tenantRepository; private TimeoutBudget timeoutBudget; @Before public void setup() { Curator curator = new MockCurator(); - TenantRepository tenants = new TenantRepository(new TestComponentRegistry.Builder() + tenantRepository = new TenantRepository(new TestComponentRegistry.Builder() .curator(curator) .build()); - tenants.addTenant(tenantName); - tenant = tenants.getTenant(tenantName); + tenantRepository.addTenant(tenantName); + tenant = tenantRepository.getTenant(tenantName); Provisioner provisioner = new SessionHandlerTest.MockProvisioner(); - applicationRepository = new ApplicationRepository(tenants, provisioner, clock); + applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock); timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); } @@ -79,15 +82,15 @@ public class ApplicationRepositoryTest { } @Test - public void createAndPrepareAndActivate() throws IOException { - PrepareResult result = deployApp(); + public void createAndPrepareAndActivate() { + PrepareResult result = deployApp(testApp); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); assertTrue(result.configChangeActions().getRestartActions().isEmpty()); } @Test - public void deleteUnusedTenants() throws IOException { - deployApp(); + public void deleteUnusedTenants() { + deployApp(testApp); assertTrue(applicationRepository.removeUnusedTenants().isEmpty()); applicationRepository.remove(applicationId()); assertEquals(tenantName, applicationRepository.removeUnusedTenants().iterator().next()); @@ -110,17 +113,48 @@ public class ApplicationRepositoryTest { assertEquals(Vtag.currentVersion, ApplicationRepository.decideVersion(regularApp, Environment.perf, targetVersion)); } + @Test + public void deleteUnusedFileReferences() { + File fileReferencesDir = Files.createTempDir(); + + // Add file reference that is not in use and should be deleted + File filereferenceDir = new File(fileReferencesDir, "foo"); + assertTrue(filereferenceDir.mkdir()); + IOUtils.writeFile(new File(filereferenceDir, "bar"), Utf8.toBytes("test")); + + tenantRepository.addTenant(tenantName); + tenant = tenantRepository.getTenant(tenantName); + Provisioner provisioner = new SessionHandlerTest.MockProvisioner(); + applicationRepository = new ApplicationRepository(tenantRepository, provisioner, clock); + timeoutBudget = new TimeoutBudget(clock, Duration.ofSeconds(60)); + + // TODO: Deploy an app with a bundle or file that will be a file reference, too much missing in test setup to get this working now + PrepareParams prepareParams = new PrepareParams.Builder().applicationId(applicationId()).ignoreValidationErrors(true).build(); + deployApp(new File("src/test/apps/app"), prepareParams); + + boolean deleteFiles = false; + Set<String> toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles); + assertEquals(new HashSet<>(Collections.singletonList("foo")), toBeDeleted); + assertTrue(filereferenceDir.exists()); + + deleteFiles = true; + toBeDeleted = applicationRepository.deleteUnusedFiledistributionReferences(fileReferencesDir, deleteFiles); + assertEquals(new HashSet<>(Collections.singletonList("foo")), toBeDeleted); + assertFalse(filereferenceDir.exists()); + } + private PrepareResult prepareAndActivateApp(File application) throws IOException { FilesApplicationPackage appDir = FilesApplicationPackage.fromFile(application); long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, appDir.getAppDir()); return applicationRepository.prepareAndActivate(tenant, sessionId, prepareParams(), false, false, Instant.now()); } - private PrepareResult deployApp() throws IOException { - File file = CompressedApplicationInputStreamTest.createTarFile(); - return applicationRepository.deploy(CompressedApplicationInputStream.createFromCompressedStream( - new FileInputStream(file), ApplicationApiHandler.APPLICATION_X_GZIP), - prepareParams(), false, false, Instant.now()); + private PrepareResult deployApp(File applicationPackage) { + return deployApp(applicationPackage, prepareParams()); + } + + private PrepareResult deployApp(File applicationPackage, PrepareParams prepareParams) { + return applicationRepository.deploy(applicationPackage, prepareParams); } private PrepareParams prepareParams() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java index de27a185aef..099cf1e1d56 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/InjectedGlobalComponentRegistryTest.java @@ -60,7 +60,7 @@ public class InjectedGlobalComponentRegistryTest { .configDefinitionsDir(Files.createTempDir().getAbsolutePath())); sessionPreparer = new SessionTest.MockSessionPreparer(); rpcServer = new RpcServer(configserverConfig, null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath())); + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(Files.createTempDir())); generationCounter = new SuperModelGenerationCounter(curator); defRepo = new StaticConfigDefinitionRepo(); permanentApplicationPackage = new PermanentApplicationPackage(configserverConfig); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java index feb3ddae4ca..e4e45d3a014 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/TestComponentRegistry.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.model.VespaModelFactory; +import java.io.File; import java.time.Clock; import java.util.Collections; import java.util.Optional; @@ -152,7 +153,7 @@ public class TestComponentRegistry implements GlobalComponentRegistry { final PermanentApplicationPackage permApp = this.permanentApplicationPackage .orElse(new PermanentApplicationPackage(configserverConfig)); FileDistributionFactory fileDistributionFactory = this.fileDistributionFactory - .orElse(new MockFileDistributionFactory()); + .orElse(new MockFileDistributionFactory(new File(configserverConfig.fileReferencesDir()))); HostProvisionerProvider hostProvisionerProvider = hostProvisioner.isPresent() ? HostProvisionerProvider.withProvisioner(hostProvisioner.get()) : HostProvisionerProvider.empty(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java index 157c36d7aef..1c0102e0b70 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; +import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.io.IOUtils; import com.yahoo.net.HostName; @@ -73,8 +74,10 @@ public class FileServerTest { @Test public void requireThatDifferentNumberOfConfigServersWork() { // Empty connection pool in tests etc. - ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder(); - FileServer fileServer = new FileServer(new ConfigserverConfig(builder)); + ConfigserverConfig.Builder builder = new ConfigserverConfig.Builder() + .configServerDBDir(Files.createTempDir().getAbsolutePath()) + .configDefinitionsDir(Files.createTempDir().getAbsolutePath()); + FileServer fileServer = createFileServer(builder); assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); // Empty connection pool when only one server, no use in downloading from yourself @@ -84,7 +87,7 @@ public class FileServerTest { serverBuilder.port(123456); servers.add(serverBuilder); builder.zookeeperserver(servers); - fileServer = new FileServer(new ConfigserverConfig(builder)); + fileServer = createFileServer(builder); assertEquals(0, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); // connection pool of size 1 when 2 servers @@ -93,10 +96,16 @@ public class FileServerTest { serverBuilder2.port(123456); servers.add(serverBuilder2); builder.zookeeperserver(servers); - fileServer = new FileServer(new ConfigserverConfig(builder)); + fileServer = createFileServer(builder); assertEquals(1, fileServer.downloader().fileReferenceDownloader().connectionPool().getSize()); } + private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) { + File fileReferencesDir = Files.createTempDir(); + configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath()); + return new FileServer(new ConfigserverConfig(configBuilder)); + } + private static class FileReceiver implements FileServer.Receiver { CompletableFuture<byte []> content; FileReceiver(CompletableFuture<byte []> content) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java index 5a9735f774a..f06f6c6e151 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpc.java @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; +import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; -import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Version; import com.yahoo.vespa.config.protocol.ConfigResponse; @@ -38,7 +38,7 @@ public class MockRpc extends RpcServer { public MockRpc(int port, boolean createDefaultTenant, boolean pretendToHaveLoadedAnyApplication) { super(createConfig(port), null, Metrics.createTestMetrics(), - new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(FileDistribution.getDefaultFileDBPath())); + new HostRegistries(), new ConfigRequestHostLivenessTracker(), new FileServer(Files.createTempDir())); if (createDefaultTenant) { onTenantCreate(TenantName.from("default"), new MockTenantProvider(pretendToHaveLoadedAnyApplication)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java index e022b622fb0..eadf17a019d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/TestWithRpc.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.rpc; +import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.config.provision.HostLivenessTracker; @@ -94,7 +95,7 @@ public class TestWithRpc { emptyNodeFlavors(), generationCounter)), Metrics.createTestMetrics(), new HostRegistries(), - hostLivenessTracker, new FileServer(FileDistribution.getDefaultFileDBPath())); + hostLivenessTracker, new FileServer(Files.createTempDir())); rpcServer.onTenantCreate(TenantName.from("default"), tenantProvider); t = new Thread(rpcServer); t.start(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java index 9d8b7c5cc00..2c05017d449 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/MockFileDistributionFactory.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; +import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.vespa.config.server.filedistribution.FileDistributionProvider; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionProvider; @@ -11,7 +12,12 @@ import java.io.File; */ public class MockFileDistributionFactory extends FileDistributionFactory { - public final MockFileDistributionProvider mockFileDistributionProvider = new MockFileDistributionProvider(); + public final MockFileDistributionProvider mockFileDistributionProvider; + + public MockFileDistributionFactory(File fileReferencesDir) { + super(new ConfigserverConfig(new ConfigserverConfig.Builder())); + mockFileDistributionProvider = new MockFileDistributionProvider(fileReferencesDir); + } @Override public FileDistributionProvider createProvider(File applicationFile) { |