diff options
author | Bjørn Christian Seime <bjorn.christian@seime.no> | 2021-01-19 19:19:33 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-01-19 19:19:33 +0100 |
commit | 3e62c6ae4a6d85699f1af6eeff533aef30493f20 (patch) | |
tree | ab57470816ce664a4cd37e5c01f90035131b3c99 /vespajlib/src/main/java/com/yahoo/compress | |
parent | 6f326f6cc056dc46f5538a8a186c8420b0379edc (diff) |
Revert "Bjorncs/zstd java"
Diffstat (limited to 'vespajlib/src/main/java/com/yahoo/compress')
4 files changed, 1 insertions, 158 deletions
diff --git a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java index 36deb318ae8..c3e42895a5e 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java +++ b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java @@ -11,8 +11,7 @@ public enum CompressionType { // Do not change the type->ordinal association. The gap is due to historic types no longer supported. NONE((byte) 0), INCOMPRESSIBLE((byte) 5), - LZ4((byte) 6), - ZSTD((byte) 7); + LZ4((byte) 6); private byte code; @@ -39,8 +38,6 @@ public enum CompressionType { return INCOMPRESSIBLE; case ((byte) 6): return LZ4; - case ((byte) 7): - return ZSTD; default: throw new IllegalArgumentException("Unknown compression type ordinal " + value); } diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index 3220b81a3a9..fb5da192f36 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -18,7 +18,6 @@ import java.util.Random; */ public class Compressor { - private final ZstdCompressor zstdCompressor = new ZstdCompressor(); private final CompressionType type; private final int level; private final double compressionThresholdFactor; @@ -92,11 +91,6 @@ public class Compressor { if (compressedData.length + 8 >= dataSize * compressionThresholdFactor) return new Compression(CompressionType.INCOMPRESSIBLE, dataSize, data); return new Compression(CompressionType.LZ4, dataSize, compressedData); - case ZSTD: - int dataLength = uncompressedSize.orElse(data.length); - if (dataLength < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, dataLength, data); - byte[] compressed = zstdCompressor.compress(data, 0, dataLength); - return new Compression(CompressionType.ZSTD, dataLength, compressed); default: throw new IllegalArgumentException(requestedCompression + " is not supported"); } @@ -136,15 +130,6 @@ public class Compressor { if (expectedCompressedSize.isPresent() && compressedSize != expectedCompressedSize.get()) throw new IllegalStateException("Compressed size mismatch. Expected " + compressedSize + ". Got " + expectedCompressedSize.get()); return uncompressedLZ4Data; - case ZSTD: - int compressedLength = expectedCompressedSize.orElseThrow(() -> new IllegalArgumentException("Zstd decompressor requires input size")); - byte[] decompressedData = zstdCompressor.decompress(compressedData, compressedDataOffset, compressedLength); - expectedCompressedSize.ifPresent(expectedSize -> { - if (compressedData.length != expectedSize) { - throw new IllegalStateException("Compressed size mismatch. Expected " + expectedSize + ". Got " + decompressedData.length); - } - }); - return decompressedData; default: throw new IllegalArgumentException(compression + " is not supported"); } diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java deleted file mode 100644 index 72ccb730db7..00000000000 --- a/vespajlib/src/main/java/com/yahoo/compress/ZstdCompressor.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.compress; - -import java.util.Arrays; - -/** - * Frame based Zstd compressor (https://github.com/facebook/zstd) - * Implemented based on https://github.com/airlift/aircompressor - a pure Java implementation (no JNI). - * - * @author bjorncs - */ -public class ZstdCompressor { - - private static final io.airlift.compress.zstd.ZstdCompressor compressor = new io.airlift.compress.zstd.ZstdCompressor(); - private static final io.airlift.compress.zstd.ZstdDecompressor decompressor = new io.airlift.compress.zstd.ZstdDecompressor(); - - public byte[] compress(byte[] input, int inputOffset, int inputLength) { - int maxCompressedLength = getMaxCompressedLength(inputLength); - byte[] output = new byte[maxCompressedLength]; - int compressedLength = compress(input, inputOffset, inputLength, output, 0, maxCompressedLength); - return Arrays.copyOf(output, compressedLength); - } - - public int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength) { - return compressor.compress(input, inputOffset, inputLength, output, outputOffset, maxOutputLength); - } - - /** - * Note: - * Implementation assumes single frame (since {@link #getDecompressedLength(byte[], int, int)} only includes the first frame) - * The {@link #decompress(byte[], int, int, byte[], int, int)} overload will try to decompress all frame, causing the output buffer to overflow. - */ - public byte[] decompress(byte[] input, int inputOffset, int inputLength) { - int decompressedLength = getDecompressedLength(input, inputOffset, inputLength); - byte[] output = new byte[decompressedLength]; - decompress(input, inputOffset, inputLength, output, 0, decompressedLength); - return output; - } - - public int decompress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset, int maxOutputLength) { - return decompressor.decompress(input, inputOffset, inputLength, output, outputOffset, maxOutputLength); - } - - public static int getMaxCompressedLength(int uncompressedLength) { - return compressor.maxCompressedLength(uncompressedLength); - } - - public static int getDecompressedLength(byte[] input, int inputOffset, int inputLength) { - return (int) io.airlift.compress.zstd.ZstdDecompressor.getDecompressedSize(input, inputOffset, inputLength); - } -} diff --git a/vespajlib/src/main/java/com/yahoo/compress/ZstdOuputStream.java b/vespajlib/src/main/java/com/yahoo/compress/ZstdOuputStream.java deleted file mode 100644 index e81bcf6a465..00000000000 --- a/vespajlib/src/main/java/com/yahoo/compress/ZstdOuputStream.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.compress; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * @author bjorncs - */ -public class ZstdOuputStream extends OutputStream { - - private final ZstdCompressor compressor = new ZstdCompressor(); - - public static final int DEFAULT_INPUT_BUFFER_SIZE = 8*1024; - - private final OutputStream out; - private final byte[] inputBuffer; - private final byte[] outputBuffer; - private int inputPosition = 0; - private boolean isClosed = false; - - public ZstdOuputStream(OutputStream out, int inputBufferSize) { - this.out = out; - this.inputBuffer = new byte[inputBufferSize]; - this.outputBuffer = new byte[ZstdCompressor.getMaxCompressedLength(inputBufferSize)]; - } - - public ZstdOuputStream(OutputStream out) { - this(out, DEFAULT_INPUT_BUFFER_SIZE); - } - - @Override - public void write(int b) throws IOException { - throwIfClosed(); - inputBuffer[inputPosition++] = (byte) b; - flushIfFull(); - } - - @Override - public void write(byte[] b) throws IOException { - throwIfClosed(); - write(b, 0, b.length); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - throwIfClosed(); - int end = off + len; - while (off < end) { - int copyLength = Math.min(end - off, inputBuffer.length - inputPosition); - System.arraycopy(b, off, inputBuffer, inputPosition, copyLength); - off += copyLength; - inputPosition += copyLength; - flushIfFull(); - } - } - - @Override - public void flush() throws IOException { - flushInternal(); - out.flush(); - } - - @Override - public void close() throws IOException { - throwIfClosed(); - flush(); - out.close(); - isClosed = true; - } - - private void flushInternal() throws IOException { - throwIfClosed(); - int compressedLength = compressor.compress(inputBuffer, 0, inputPosition, outputBuffer, 0, outputBuffer.length); - out.write(outputBuffer, 0, compressedLength); - inputPosition = 0; - } - - private void flushIfFull() throws IOException { - if (inputPosition == inputBuffer.length) { - flushInternal(); - } - } - - private void throwIfClosed() { - if (isClosed) throw new IllegalArgumentException("Output stream is already closed"); - } -} |