diff options
11 files changed, 55 insertions, 20 deletions
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index d3b7e53157d..7f120a88a05 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -141,7 +141,7 @@ public class FileDirectory { File destination = new File(tempDestinationDir.toFile(), source.getName()); if (!destinationDir.exists()) { destinationDir.mkdir(); - log.log(Level.FINE, () -> "file reference ' " + reference.value() + "', source: " + source.getAbsolutePath() ); + log.log(Level.FINE, () -> "file reference '" + reference.value() + "', source: " + source.getAbsolutePath() ); if (source.isDirectory()) { log.log(Level.FINE, () -> "Copying source " + source.getAbsolutePath() + " to " + destination.getAbsolutePath()); IOUtils.copyDirectory(source, destination, -1); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 179d8923378..770352e6bfc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -44,6 +44,7 @@ import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUti import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type; public class FileServer { @@ -154,9 +155,9 @@ public class FileServer { CompressionType compressionType = chooseCompressionType(acceptedCompressionTypes); log.log(Level.FINE, () -> "accepted compression types=" + acceptedCompressionTypes + ", compression type to use=" + compressionType); File compressedFile = new FileReferenceCompressor(compressed, compressionType).compress(file.getParentFile(), tempFile.toFile()); - return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile); + return new LazyTemporaryStorageFileReferenceData(reference, file.getName(), compressed, compressedFile, compressionType); } else { - return new LazyFileReferenceData(reference, file.getName(), FileReferenceData.Type.file, file); + return new LazyFileReferenceData(reference, file.getName(), Type.file, file, gzip); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 687cb1d3cca..92bb44a3654 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -491,6 +491,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { sendParts(session, fileData); sendEof(session, fileData, status); } + private void sendParts(int session, FileReferenceData fileData) { ByteBuffer bb = ByteBuffer.allocate(0x100000); for (int partId = 0, read = fileData.nextContent(bb); read >= 0; partId++, read = fileData.nextContent(bb)) { @@ -504,12 +505,9 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { bb.clear(); } } + private int sendMeta(FileReferenceData fileData) { - Request request = new Request(FileReceiver.RECEIVE_META_METHOD); - request.parameters().add(new StringValue(fileData.fileReference().value())); - request.parameters().add(new StringValue(fileData.filename())); - request.parameters().add(new StringValue(fileData.type().name())); - request.parameters().add(new Int64Value(fileData.size())); + Request request = createMetaRequest(fileData); invokeRpcIfValidConnection(request); if (request.isError()) { log.warning("Failed delivering meta for reference '" + fileData.fileReference().value() + "' with file '" + fileData.filename() + "' to " + @@ -522,6 +520,18 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { return request.returnValues().get(1).asInt32(); } } + + // non-private for testing + static Request createMetaRequest(FileReferenceData fileData) { + Request request = new Request(FileReceiver.RECEIVE_META_METHOD); + request.parameters().add(new StringValue(fileData.fileReference().value())); + request.parameters().add(new StringValue(fileData.filename())); + request.parameters().add(new StringValue(fileData.type().name())); + request.parameters().add(new Int64Value(fileData.size())); + request.parameters().add(new StringValue(fileData.compressionType().name())); + return request; + } + private void sendPart(int session, FileReference ref, int partId, byte [] buf) { Request request = new Request(FileReceiver.RECEIVE_PART_METHOD); request.parameters().add(new StringValue(ref.value())); @@ -538,6 +548,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } } } + private void sendEof(int session, FileReferenceData fileData, FileServer.ReplayStatus status) { Request request = new Request(FileReceiver.RECEIVE_EOF_METHOD); request.parameters().add(new StringValue(fileData.fileReference().value())); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 7ad237e45ed..fe3bf4a2a29 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.LbServicesConfig; import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.component.Version; +import com.yahoo.config.FileReference; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; @@ -27,16 +28,19 @@ import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.session.PrepareParams; +import com.yahoo.vespa.filedistribution.LazyFileReferenceData; import com.yahoo.vespa.model.VespaModel; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; - import java.io.File; import java.io.IOException; import java.util.Optional; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; +import static com.yahoo.vespa.config.server.rpc.RpcServer.ChunkedFileReceiver.createMetaRequest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -97,6 +101,17 @@ public class RpcServerTest { } } + @Test + public void testFileReceiverMetaRequest() throws IOException { + File file = temporaryFolder.newFile(); + Request request = createMetaRequest(new LazyFileReferenceData(new FileReference("foo"), "fileA", compressed, file, lz4)); + assertEquals("foo", request.parameters().get(0).asString()); + assertEquals("fileA", request.parameters().get(1).asString()); + assertEquals("compressed", request.parameters().get(2).asString()); + assertEquals(0, request.parameters().get(3).asInt64()); + assertEquals("lz4", request.parameters().get(4).asString()); + } + private JRTClientConfigRequest createSimpleRequest() { ConfigKey<?> key = new ConfigKey<>(SimpletypesConfig.class, ""); JRTClientConfigRequest clientReq = createRequest(new RawConfig(key, SimpletypesConfig.getDefMd5())); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java index 8b3bc32ff71..ea8461b42f3 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/EmptyFileReferenceData.java @@ -12,7 +12,7 @@ public class EmptyFileReferenceData extends FileReferenceData { private int contentRead = 0; private EmptyFileReferenceData(FileReference fileReference, String filename, Type type, byte[] content, long xxhash) { - super(fileReference, filename, type); + super(fileReference, filename, type, CompressionType.gzip); this.content = content; this.xxhash = xxhash; } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index 65c6dd5931d..a285fbaafe2 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -129,6 +129,7 @@ public class FileReceiver { moveFileToDestination(inprogressFile, file); } else { decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); + log.log(Level.FINE, () -> "compression type to use=" + compressionType); new FileReferenceCompressor(fileType, compressionType).decompress(inprogressFile, decompressedDir); moveFileToDestination(decompressedDir, fileReferenceDir); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index b485e6ded86..efb845bafe7 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -118,9 +118,9 @@ public class FileReferenceCompressor { } private OutputStream compressedOutputStream(File outputFile) throws IOException { - log.log(Level.FINE, () -> "Compressing with type " + type + " and compression type " + compressionType); switch (type) { case compressed: + log.log(Level.FINE, () -> "Compressing with compression type " + compressionType); switch (compressionType) { case gzip: return new GZIPOutputStream(new FileOutputStream(outputFile)); @@ -137,9 +137,9 @@ public class FileReferenceCompressor { } private InputStream decompressedInputStream(File inputFile) throws IOException { - log.log(Level.FINE, () -> "Decompressing with type " + type + " and compression type " + compressionType); switch (type) { case compressed: + log.log(Level.FINE, () -> "Decompressing with compression type " + compressionType); switch (compressionType) { case gzip: return new GZIPInputStream(new FileInputStream(inputFile)); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java index d14f690b2d3..3f83cbea506 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java @@ -18,11 +18,13 @@ public abstract class FileReferenceData { private final FileReference fileReference; private final String filename; private final Type type; + private final CompressionType compressionType; - public FileReferenceData(FileReference fileReference, String filename, Type type) { + public FileReferenceData(FileReference fileReference, String filename, Type type, CompressionType compressionType) { this.fileReference = fileReference; this.filename = filename; this.type = type; + this.compressionType = compressionType; } public FileReference fileReference() {return fileReference;} @@ -31,6 +33,8 @@ public abstract class FileReferenceData { public Type type() {return type;} + public CompressionType compressionType() { return compressionType;} + public ByteBuffer content() { ByteBuffer bb = ByteBuffer.allocate((int)size()); while (bb.remaining() > 0) { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java index 10de9c072b9..c8fbb639b35 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyFileReferenceData.java @@ -17,8 +17,8 @@ public class LazyFileReferenceData extends FileReferenceData { private final ReadableByteChannel channel; private final StreamingXXHash64 hasher; - public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException { - super(fileReference, filename, type); + public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file, CompressionType compressionType) throws IOException { + super(fileReference, filename, type, compressionType); this.file = file; channel = Files.newByteChannel(file.toPath()); this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java index 974d5ff1489..a69370b28ff 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/LazyTemporaryStorageFileReferenceData.java @@ -12,8 +12,8 @@ import java.nio.file.Files; */ public class LazyTemporaryStorageFileReferenceData extends LazyFileReferenceData { - public LazyTemporaryStorageFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException { - super(fileReference, filename, type, file); + public LazyTemporaryStorageFileReferenceData(FileReference fileReference, String filename, Type type, File file, CompressionType compressionType) throws IOException { + super(fileReference, filename, type, file, compressionType); } public void close() { diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java index 66b731f204b..eda93331c73 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileReferenceDataTest.java @@ -7,12 +7,15 @@ import com.yahoo.text.Utf8; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType; +import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type; +import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -27,7 +30,7 @@ public class FileReferenceDataTest { String content = "blob"; File tempFile = writeTempFile(content); FileReferenceData fileReferenceData = - new LazyTemporaryStorageFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, tempFile); + new LazyTemporaryStorageFileReferenceData(new FileReference("ref"), "foo", compressed, tempFile, gzip); ByteBuffer byteBuffer = ByteBuffer.allocate(100); assertEquals(4, fileReferenceData.nextContent(byteBuffer)); assertEquals(content, Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 4))); @@ -44,7 +47,7 @@ public class FileReferenceDataTest { String content = "blobbblubbblabb"; File file = writeTempFile(content); FileReferenceData fileReferenceData = - new LazyFileReferenceData(new FileReference("ref"), "foo", FileReferenceData.Type.compressed, file); + new LazyFileReferenceData(new FileReference("ref"), "foo", Type.compressed, file, CompressionType.gzip); ByteBuffer byteBuffer = ByteBuffer.allocate(10); assertEquals(10, fileReferenceData.nextContent(byteBuffer)); assertEquals(content.substring(0,10), Utf8.toString(Arrays.copyOfRange(byteBuffer.array(), 0, 10))); |