Support compressing files (not just directories) in file distribution

Use file distribution compression feature flags and support compressing single
files

4 files changed, 44 insertions, 23 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
index 1edcd3c4d31..14d99cc17af 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
@@ -13,14 +13,18 @@ import com.yahoo.jrt.Supervisor;
 import com.yahoo.jrt.Transport;
 import com.yahoo.vespa.config.ConnectionPool;
 import com.yahoo.vespa.defaults.Defaults;
-import com.yahoo.vespa.filedistribution.FileReferenceCompressor;
 import com.yahoo.vespa.filedistribution.EmptyFileReferenceData;
 import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool;
 import com.yahoo.vespa.filedistribution.FileDownloader;
+import com.yahoo.vespa.filedistribution.FileReferenceCompressor;
 import com.yahoo.vespa.filedistribution.FileReferenceData;
 import com.yahoo.vespa.filedistribution.FileReferenceDownload;
 import com.yahoo.vespa.filedistribution.LazyFileReferenceData;
 import com.yahoo.vespa.filedistribution.LazyTemporaryStorageFileReferenceData;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.StringFlag;
 import com.yahoo.yolean.Exceptions;
 import java.io.File;
 import java.io.IOException;
@@ -36,7 +40,6 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getOtherConfigServersInCluster;
-import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed;
 
 public class FileServer {
 
@@ -48,6 +51,8 @@ public class FileServer {
     private final FileDirectory root;
     private final ExecutorService executor;
     private final FileDownloader downloader;
+    private final StringFlag compressionAlgorithm;
+    private final BooleanFlag compressSingleFiles;
 
     private enum FileApiErrorCodes {
         OK(0, "OK"),
@@ -80,21 +85,24 @@ public class FileServer {
 
     @SuppressWarnings("WeakerAccess") // Created by dependency injection
     @Inject
-    public FileServer(ConfigserverConfig configserverConfig) {
+    public FileServer(ConfigserverConfig configserverConfig, FlagSource flagSource) {
         this(new File(Defaults.getDefaults().underVespaHome(configserverConfig.fileReferencesDir())),
-             createFileDownloader(getOtherConfigServersInCluster(configserverConfig)));
+             createFileDownloader(getOtherConfigServersInCluster(configserverConfig)),
+             flagSource);
     }
 
     // For testing only
-    public FileServer(File rootDir) {
-        this(rootDir, createFileDownloader(List.of()));
+    public FileServer(File rootDir, FlagSource flagSource) {
+        this(rootDir, createFileDownloader(List.of()), flagSource);
     }
 
-    public FileServer(File rootDir, FileDownloader fileDownloader) {
+    public FileServer(File rootDir, FileDownloader fileDownloader, FlagSource flagSource) {
         this.downloader = fileDownloader;
         this.root = new FileDirectory(rootDir);
         this.executor = Executors.newFixedThreadPool(Math.max(8, Runtime.getRuntime().availableProcessors()),
                                                      new DaemonThreadFactory("file-server-"));
+        this.compressionAlgorithm = Flags.FILE_DISTRIBUTION_COMPRESSION_ALGORITHM.bindTo(flagSource);
+        this.compressSingleFiles = Flags.FILE_DISTRIBUTION_COMPRESS_SINGLE_FILES.bindTo(flagSource);
     }
 
     boolean hasFile(String fileReference) {
@@ -147,17 +155,27 @@ public class FileServer {
     }
 
     private FileReferenceData readFileReferenceData(FileReference reference) throws IOException {
+        FileReferenceData.Type type = FileReferenceData.from(compressionAlgorithm.value());
         File file = root.getFile(reference);
-
+        String fileName = file.getName();
         if (file.isDirectory()) {
-            Path tempFile = Files.createTempFile("filereferencedata", reference.value());
-            File compressedFile = new FileReferenceCompressor(compressed).compress(file.getParentFile(), tempFile.toFile());
-            return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile);
+            return createFileReferenceData(file.getParentFile(), reference, type, fileName);
+        } else if (compressSingleFiles.value()) {
+            return createFileReferenceData(file, reference, type, fileName);
         } else {
-            return new LazyFileReferenceData(reference, file.getName(), FileReferenceData.Type.file, file);
+            return new LazyFileReferenceData(reference, fileName, FileReferenceData.Type.file, file);
         }
     }
 
+    LazyTemporaryStorageFileReferenceData createFileReferenceData(File file,
+                                                                  FileReference reference,
+                                                                  FileReferenceData.Type type,
+                                                                  String fileName) throws IOException {
+        Path tempFile = Files.createTempFile("filereferencedata", reference.value());
+        File compressedFile = new FileReferenceCompressor(type).compress(file.getParentFile(), tempFile.toFile());
+        return new LazyTemporaryStorageFileReferenceData(reference, fileName, type, compressedFile);
+    }
+
     public void serveFile(String fileReference, boolean downloadFromOtherSourceIfNotFound, Request request, Receiver receiver) {
         if (executor instanceof ThreadPoolExecutor)
             log.log(Level.FINE, () -> "Active threads: " + ((ThreadPoolExecutor) executor).getActiveCount());
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
index 67c40f94b6a..216a81af0ab 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileServerTest.java
@@ -10,6 +10,7 @@ import com.yahoo.net.HostName;
 import com.yahoo.vespa.filedistribution.FileDownloader;
 import com.yahoo.vespa.filedistribution.FileReferenceData;
 import com.yahoo.vespa.filedistribution.FileReferenceDownload;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -37,7 +38,7 @@ public class FileServerTest {
     @Before
     public void setup() throws IOException {
         File rootDir = new File(temporaryFolder.newFolder("fileserver-root").getAbsolutePath());
-        fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir));
+        fileServer = new FileServer(rootDir, new MockFileDownloader(rootDir), new InMemoryFlagSource());
     }
 
     @Test
@@ -117,7 +118,7 @@ public class FileServerTest {
     private FileServer createFileServer(ConfigserverConfig.Builder configBuilder) throws IOException {
         File fileReferencesDir = temporaryFolder.newFolder();
         configBuilder.fileReferencesDir(fileReferencesDir.getAbsolutePath());
-        return new FileServer(new ConfigserverConfig(configBuilder));
+        return new FileServer(new ConfigserverConfig(configBuilder), new InMemoryFlagSource());
     }
 
     private static class FileReceiver implements FileServer.Receiver {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java
index 3272689473e..77f67604aad 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/MockRpcServer.java
@@ -11,6 +11,7 @@ import com.yahoo.vespa.config.server.host.ConfigRequestHostLivenessTracker;
 import com.yahoo.vespa.config.server.host.HostRegistry;
 import com.yahoo.vespa.config.server.monitoring.Metrics;
 import com.yahoo.vespa.config.server.rpc.security.NoopRpcAuthorizer;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 
 import java.io.File;
 import java.time.Duration;
@@ -39,7 +40,7 @@ public class MockRpcServer extends RpcServer {
               Metrics.createTestMetrics(),
               new HostRegistry(),
               new ConfigRequestHostLivenessTracker(),
-              new FileServer(tempDir),
+              new FileServer(tempDir, new InMemoryFlagSource()),
               new NoopRpcAuthorizer(),
               new RpcRequestHandlerProvider());
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java
index 40ed20b7969..73312710c45 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java
@@ -112,20 +112,21 @@ public class RpcTester implements AutoCloseable {
     }
 
     RpcServer createRpcServer(ConfigserverConfig config) throws IOException {
+        InMemoryFlagSource flagSource = new InMemoryFlagSource();
         RpcServer rpcServer = new RpcServer(config,
-                             new SuperModelRequestHandler(new TestConfigDefinitionRepo(),
+                                            new SuperModelRequestHandler(new TestConfigDefinitionRepo(),
                                                           configserverConfig,
                                                           new SuperModelManager(
                                                                   config,
                                                                   Zone.defaultZone(),
                                                                   new MemoryGenerationCounter(),
-                                                                  new InMemoryFlagSource())),
-                             Metrics.createTestMetrics(),
-                             hostRegistry,
-                             hostLivenessTracker,
-                             new FileServer(temporaryFolder.newFolder()),
-                             new NoopRpcAuthorizer(),
-                             new RpcRequestHandlerProvider());
+                                                                  flagSource)),
+                                            Metrics.createTestMetrics(),
+                                            hostRegistry,
+                                            hostLivenessTracker,
+                                            new FileServer(temporaryFolder.newFolder(), flagSource),
+                                            new NoopRpcAuthorizer(),
+                                            new RpcRequestHandlerProvider());
         rpcServer.setUpGetConfigHandlers();
         return rpcServer;
     }