Make sure to set compression type in file distribution meta request

11 files changed, 55 insertions, 20 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
index d3b7e53157d..7f120a88a05 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
@@ -141,7 +141,7 @@ public class FileDirectory  {
             File destination = new File(tempDestinationDir.toFile(), source.getName());
             if (!destinationDir.exists()) {
                 destinationDir.mkdir();
-                log.log(Level.FINE, () -> "file reference ' " + reference.value() + "', source: " + source.getAbsolutePath() );
+                log.log(Level.FINE, () -> "file reference '" + reference.value() + "', source: " + source.getAbsolutePath() );
                 if (source.isDirectory()) {
                     log.log(Level.FINE, () -> "Copying source " + source.getAbsolutePath() + " to " + destination.getAbsolutePath());
                     IOUtils.copyDirectory(source, destination, -1);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
index 179d8923378..770352e6bfc 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java
@@ -44,6 +44,7 @@ import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUti
 import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType;
 import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip;
 import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed;
+import static com.yahoo.vespa.filedistribution.FileReferenceData.Type;
 
 public class FileServer {
 
@@ -154,9 +155,9 @@ public class FileServer {
             CompressionType compressionType = chooseCompressionType(acceptedCompressionTypes);
             log.log(Level.FINE, () -> "accepted compression types=" + acceptedCompressionTypes + ", compression type to use=" + compressionType);
             File compressedFile = new FileReferenceCompressor(compressed, compressionType).compress(file.getParentFile(), tempFile.toFile());
-            return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile);
+            return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile, compressionType);
         } else {
-            return new LazyFileReferenceData(reference, file.getName(), FileReferenceData.Type.file, file);
+            return new LazyFileReferenceData(reference, file.getName(), Type.file, file, gzip);
         }
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
index 687cb1d3cca..92bb44a3654 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
@@ -491,6 +491,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
             sendParts(session, fileData);
             sendEof(session, fileData, status);
         }
+
         private void sendParts(int session, FileReferenceData fileData) {
             ByteBuffer bb = ByteBuffer.allocate(0x100000);
             for (int partId = 0, read = fileData.nextContent(bb); read >= 0; partId++, read = fileData.nextContent(bb)) {
@@ -504,12 +505,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
                 bb.clear();
             }
         }
+
         private int sendMeta(FileReferenceData fileData) {
-            Request request = new Request(FileReceiver.RECEIVE_META_METHOD);
-            request.parameters().add(new StringValue(fileData.fileReference().value()));
-            request.parameters().add(new StringValue(fileData.filename()));
-            request.parameters().add(new StringValue(fileData.type().name()));
-            request.parameters().add(new Int64Value(fileData.size()));
+            Request request = createMetaRequest(fileData);
             invokeRpcIfValidConnection(request);
             if (request.isError()) {
                 log.warning("Failed delivering meta for reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " +
@@ -522,6 +520,18 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
                 return request.returnValues().get(1).asInt32();
             }
         }
+
+        // non-private for testing
+        static Request createMetaRequest(FileReferenceData fileData) {
+            Request request = new Request(FileReceiver.RECEIVE_META_METHOD);
+            request.parameters().add(new StringValue(fileData.fileReference().value()));
+            request.parameters().add(new StringValue(fileData.filename()));
+            request.parameters().add(new StringValue(fileData.type().name()));
+            request.parameters().add(new Int64Value(fileData.size()));
+            request.parameters().add(new StringValue(fileData.compressionType().name()));
+            return request;
+        }
+
         private void sendPart(int session, FileReference ref, int partId, byte [] buf) {
             Request request = new Request(FileReceiver.RECEIVE_PART_METHOD);
             request.parameters().add(new StringValue(ref.value()));
@@ -538,6 +548,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener {
                 }
             }
         }
+
         private void sendEof(int session, FileReferenceData fileData, FileServer.ReplayStatus status) {
             Request request = new Request(FileReceiver.RECEIVE_EOF_METHOD);
             request.parameters().add(new StringValue(fileData.fileReference().value()));
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
index 7ad237e45ed..fe3bf4a2a29 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java
@@ -5,6 +5,7 @@ import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.cloud.config.LbServicesConfig;
 import com.yahoo.cloud.config.SentinelConfig;
 import com.yahoo.component.Version;
+import com.yahoo.config.FileReference;
 import com.yahoo.config.SimpletypesConfig;
 import com.yahoo.config.model.test.MockApplicationPackage;
 import com.yahoo.config.provision.ApplicationId;
@@ -27,16 +28,19 @@ import com.yahoo.vespa.config.server.application.Application;
 import com.yahoo.vespa.config.server.application.ApplicationSet;
 import com.yahoo.vespa.config.server.monitoring.MetricUpdater;
 import com.yahoo.vespa.config.server.session.PrepareParams;
+import com.yahoo.vespa.filedistribution.LazyFileReferenceData;
 import com.yahoo.vespa.model.VespaModel;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 import org.xml.sax.SAXException;
-
 import java.io.File;
 import java.io.IOException;
 import java.util.Optional;
 
+import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4;
+import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed;
+import static com.yahoo.vespa.config.server.rpc.RpcServer.ChunkedFileReceiver.createMetaRequest;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
@@ -97,6 +101,17 @@ public class RpcServerTest {
         }
     }
 
+    @Test
+    public void testFileReceiverMetaRequest() throws IOException {
+        File file = temporaryFolder.newFile();
+        Request request = createMetaRequest(new LazyFileReferenceData(new FileReference("foo"), "fileA", compressed, file, lz4));
+        assertEquals("foo", request.parameters().get(0).asString());
+        assertEquals("fileA", request.parameters().get(1).asString());
+        assertEquals("compressed", request.parameters().get(2).asString());
+        assertEquals(0, request.parameters().get(3).asInt64());
+        assertEquals("lz4", request.parameters().get(4).asString());
+    }
+
     private JRTClientConfigRequest createSimpleRequest() {
         ConfigKey<?> key = new ConfigKey<>(SimpletypesConfig.class, "");
         JRTClientConfigRequest clientReq = createRequest(new RawConfig(key, SimpletypesConfig.getDefMd5()));
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java
index 8b3bc32ff71..ea8461b42f3 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java
@@ -12,7 +12,7 @@ public class EmptyFileReferenceData extends FileReferenceData {
     private int contentRead = 0;
 
     private EmptyFileReferenceData(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) {
-        super(fileReference, filename, type);
+        super(fileReference, filename, type, CompressionType.gzip);
         this.content = content;
         this.xxhash = xxhash;
     }
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
index 65c6dd5931d..a285fbaafe2 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java
@@ -129,6 +129,7 @@ public class FileReceiver {
                     moveFileToDestination(inprogressFile, file);
                 } else {
                     decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile();
+                    log.log(Level.FINE, () -> "compression type to use=" + compressionType);
                     new FileReferenceCompressor(fileType, compressionType).decompress(inprogressFile, decompressedDir);
                     moveFileToDestination(decompressedDir, fileReferenceDir);
                 }
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java
index b485e6ded86..efb845bafe7 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java
@@ -118,9 +118,9 @@ public class FileReferenceCompressor {
     }
 
     private OutputStream compressedOutputStream(File outputFile) throws IOException {
-        log.log(Level.FINE, () -> "Compressing with type " + type + " and compression type " + compressionType);
         switch (type) {
             case compressed:
+                log.log(Level.FINE, () -> "Compressing with compression type " + compressionType);
                 switch (compressionType) {
                     case gzip:
                         return new GZIPOutputStream(new FileOutputStream(outputFile));
@@ -137,9 +137,9 @@ public class FileReferenceCompressor {
     }
 
     private InputStream decompressedInputStream(File inputFile) throws IOException {
-        log.log(Level.FINE, () -> "Decompressing with type " + type + " and compression type " + compressionType);
         switch (type) {
             case compressed:
+                log.log(Level.FINE, () -> "Decompressing with compression type " + compressionType);
                 switch (compressionType) {
                     case gzip:
                         return new GZIPInputStream(new FileInputStream(inputFile));
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java
index d14f690b2d3..3f83cbea506 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java
@@ -18,11 +18,13 @@ public abstract class FileReferenceData {
     private final FileReference fileReference;
     private final String filename;
     private final Type type;
+    private final CompressionType compressionType;
 
-    public FileReferenceData(FileReference fileReference, String filename, Type type) {
+    public FileReferenceData(FileReference fileReference, String filename, Type type, CompressionType compressionType) {
         this.fileReference = fileReference;
         this.filename = filename;
         this.type = type;
+        this.compressionType = compressionType;
     }
 
     public FileReference fileReference() {return fileReference;}
@@ -31,6 +33,8 @@ public abstract class FileReferenceData {
 
     public Type type() {return type;}
 
+    public CompressionType compressionType() { return compressionType;}
+
     public ByteBuffer content() {
         ByteBuffer bb = ByteBuffer.allocate((int)size());
         while (bb.remaining() > 0) {
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java
index 10de9c072b9..c8fbb639b35 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java
@@ -17,8 +17,8 @@ public class LazyFileReferenceData extends FileReferenceData {
     private final ReadableByteChannel channel;
     private final StreamingXXHash64 hasher;
 
-    public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException {
-        super(fileReference, filename, type);
+    public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file, CompressionType compressionType) throws IOException {
+        super(fileReference, filename, type, compressionType);
         this.file = file;
         channel = Files.newByteChannel(file.toPath());
         this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0);
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java
index 974d5ff1489..a69370b28ff 100644
--- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java
+++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java
@@ -12,8 +12,8 @@ import java.nio.file.Files;
  */
 public class LazyTemporaryStorageFileReferenceData extends LazyFileReferenceData {
 
-    public LazyTemporaryStorageFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException {
-        super(fileReference, filename, type, file);
+    public LazyTemporaryStorageFileReferenceData(FileReference fileReference, String filename, Type type, File file, CompressionType compressionType) throws IOException {
+        super(fileReference, filename, type, file, compressionType);
     }
 
     public void close() {
diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java
index 66b731f204b..eda93331c73 100644
--- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java
+++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java
@@ -7,12 +7,15 @@ import com.yahoo.text.Utf8;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
-
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
+import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType;
+import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip;
+import static com.yahoo.vespa.filedistribution.FileReferenceData.Type;
+import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -27,7 +30,7 @@ public class FileReferenceDataTest {
         String content = "blob";
         File tempFile = writeTempFile(content);
         FileReferenceData fileReferenceData =
-                new LazyTemporaryStorageFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, tempFile);
+                new LazyTemporaryStorageFileReferenceData(new FileReference("ref"), "foo", compressed, tempFile, gzip);
         ByteBuffer byteBuffer = ByteBuffer.allocate(100);
         assertEquals(4, fileReferenceData.nextContent(byteBuffer));
         assertEquals(content, Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 4)));
@@ -44,7 +47,7 @@ public class FileReferenceDataTest {
         String content = "blobbblubbblabb";
         File file = writeTempFile(content);
         FileReferenceData fileReferenceData =
-                new LazyFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, file);
+                new LazyFileReferenceData(new FileReference("ref"), "foo", Type.compressed, file, CompressionType.gzip);
         ByteBuffer byteBuffer = ByteBuffer.allocate(10);
         assertEquals(10, fileReferenceData.nextContent(byteBuffer));
         assertEquals(content.substring(0,10), Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 10)));