diff options
5 files changed, 74 insertions, 3 deletions
diff --git a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java index 688d6d54888..03e08e039ae 100644 --- a/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java +++ b/config/src/main/java/com/yahoo/vespa/config/LZ4PayloadCompressor.java @@ -5,6 +5,8 @@ import com.yahoo.compress.CompressionType; import com.yahoo.compress.Compressor; import com.yahoo.vespa.config.util.ConfigUtils; +import java.nio.ByteBuffer; + /** * Wrapper for LZ4 compression that selects compression level based on properties. * @@ -24,9 +26,17 @@ public class LZ4PayloadCompressor { public byte[] compress(byte[] input) { return compressor.compressUnconditionally(input); } + public byte[] compress(ByteBuffer input) { + return compressor.compressUnconditionally(input); + } public byte [] decompress(byte[] input, int uncompressedLen) { return compressor.decompressUnconditionally(input, 0, uncompressedLen); } + public byte [] decompress(ByteBuffer input, int uncompressedLen) { + ByteBuffer uncompressed = ByteBuffer.allocate(uncompressedLen); + compressor.decompressUnconditionally(input, uncompressed); + return uncompressed.array(); + } } diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index 3220b81a3a9..ad51ac1ca38 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -6,6 +6,7 @@ import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FastDecompressor; import net.jpountz.lz4.LZ4SafeDecompressor; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Optional; import java.util.Random; @@ -159,7 +160,16 @@ public class Compressor { } public byte[] compressUnconditionally(byte[] input) { - return getCompressor().compress(input); + return getCompressor().compress(input, 0, input.length); + } + public byte[] compressUnconditionally(ByteBuffer input) { + return getCompressor().compress(input.array(), input.arrayOffset()+input.position(), input.remaining()); + } + + public void decompressUnconditionally(ByteBuffer input, ByteBuffer output) { + if (input.remaining() > 0) { + factory.fastDecompressor().decompress(input, output); + } } public byte [] decompressUnconditionally(byte[] input, int srcOffset, int uncompressedLen) { diff --git a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java index de9c88a713d..a14e448f922 100644 --- a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java +++ b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java @@ -28,7 +28,7 @@ public class ByteWriter extends AbstractByteWriter { @Override public void send(final ByteBuffer b) throws IOException { // we know from how BufferChain works we have a backing array - stream.write(b.array(), b.position() + b.arrayOffset(), b.limit() - b.position()); + stream.write(b.array(), b.position() + b.arrayOffset(), b.remaining()); } @Override diff --git a/vespajlib/src/main/java/com/yahoo/io/Utf8ByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/Utf8ByteWriter.java new file mode 100644 index 00000000000..729374d253f --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/Utf8ByteWriter.java @@ -0,0 +1,47 @@ +package com.yahoo.io; + +import com.yahoo.text.Utf8; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public class Utf8ByteWriter extends AbstractByteWriter { + private ByteBuffer myBuf; + public Utf8ByteWriter(int initialBuffer) { + super(Utf8.getNewEncoder()); + myBuf = ByteBuffer.allocate(initialBuffer); + } + @Override + public void send(ByteBuffer src) throws IOException { + if (myBuf.remaining() < src.remaining()) { + ByteBuffer newBuf = ByteBuffer.allocate(1 << Integer.highestOneBit(myBuf.position()+src.remaining())); + myBuf.flip(); + newBuf.put(myBuf); + myBuf = newBuf; + } + myBuf.put(src); + } + + @Override + public void flush() throws IOException { + buffer.flush(); + } + + @Override + public void close() throws IOException { + buffer.flush(); + myBuf.flip(); + } + + /** + * Return a buffer ready for read. Must only be called after writer has been closed. + * @return A flipped ByteBuffer + */ + public ByteBuffer getBuf() { + if (myBuf.position() != 0) { + throw new IllegalStateException("Call close() befor getBuf(), pos=" + myBuf.position() + ", limit=" + myBuf.limit()); + } + return myBuf; + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java index f677ae23a45..b7f11f53cd5 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonDecoder.java @@ -5,6 +5,7 @@ import com.yahoo.text.Text; import com.yahoo.text.Utf8; import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; /** * A port of the C++ json decoder intended to be fast. @@ -34,7 +35,10 @@ public class JsonDecoder { public JsonDecoder() {} public Slime decode(Slime slime, byte[] bytes) { - in = new BufferedInput(bytes); + return decode(slime, ByteBuffer.wrap(bytes)); + } + public Slime decode(Slime slime, ByteBuffer buf) { + in = new BufferedInput(buf.array(), buf.arrayOffset()+buf.position(), buf.remaining()); next(); decodeValue(slimeInserter.adjust(slime)); if (in.failed()) { |