diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-08-30 20:52:50 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-08-30 20:52:50 +0200 |
commit | 26b09e283a21d3e123f63db041d16e2eae0a54e0 (patch) | |
tree | f7f31ba59b708ee9c93a48629f3d1f24a32a886f /vespajlib | |
parent | 193dc7fa8cf4b4d5c8ca9cb1a1836821e2069d87 (diff) |
Revert "Avoid copying data just to compress them when it is not necessary."
Diffstat (limited to 'vespajlib')
7 files changed, 59 insertions, 94 deletions
diff --git a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java index 1b20f1dcaf7..98228292cc7 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/Compressor.java +++ b/vespajlib/src/main/java/com/yahoo/compress/Compressor.java @@ -87,38 +87,31 @@ public class Compressor { public Compression compress(CompressionType requestedCompression, byte[] data, int offset, int len) { switch (requestedCompression) { case NONE: - return compact(CompressionType.NONE, data, offset, len); + if ((offset != 0) || (len != data.length)) { + data = Arrays.copyOfRange(data, offset, offset + len); + } + return new Compression(CompressionType.NONE, data.length, data); case LZ4: if (len < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, len, data); byte[] compressedData = getCompressor().compress(data, offset, len); - if (compressedData.length + 8 >= len * compressionThresholdFactor) { - return compact(CompressionType.INCOMPRESSIBLE, data, offset, len); - } + if (compressedData.length + 8 >= len * compressionThresholdFactor) + return new Compression(CompressionType.INCOMPRESSIBLE, len, data); return new Compression(CompressionType.LZ4, len, compressedData); case ZSTD: - if (len < compressMinSizeBytes) { - return compact(CompressionType.INCOMPRESSIBLE, data, offset, len); - } + if (len < compressMinSizeBytes) return new Compression(CompressionType.INCOMPRESSIBLE, len, data); byte[] compressed = zstdCompressor.compress(data, offset, len); return new Compression(CompressionType.ZSTD, len, compressed); default: throw new IllegalArgumentException(requestedCompression + " is not supported"); } } - - private Compression compact(CompressionType type, byte[] data, int offset, int len) { - if ((offset != 0) || (len != data.length)) { - data = Arrays.copyOfRange(data, offset, offset + len); - } - return new Compression(type, len, data); - } private LZ4Compressor getCompressor() { return level < 7 ? factory.fastCompressor() : factory.highCompressor(); } /** Compresses some data using the requested compression type */ - public Compression compress(CompressionType requestedCompression, byte[] data) { return compress(requestedCompression, data, 0, data.length); } + public Compression compress(CompressionType requestedCompression, byte[] data) { return compress(requestedCompression, data, Optional.empty()); } /** Compresses some data using the compression type of this compressor */ - public Compression compress(byte[] data, int uncompressedSize) { return compress(type, data, 0, uncompressedSize); } + public Compression compress(byte[] data, int uncompressedSize) { return compress(type, data, Optional.of(uncompressedSize)); } /** Compresses some data using the compression type of this compressor */ public Compression compress(byte[] data) { return compress(type, data, Optional.empty()); } diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java index 13e08484d7c..ecee5e9504e 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java @@ -1,11 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.slime; - -import static com.yahoo.slime.BinaryFormat.decode_double; -import static com.yahoo.slime.BinaryFormat.decode_meta; -import static com.yahoo.slime.BinaryFormat.decode_type; -import static com.yahoo.slime.BinaryFormat.decode_zigzag; +import static com.yahoo.slime.BinaryFormat.*; final class BinaryDecoder { BufferedInput in; @@ -139,7 +135,9 @@ final class BinaryDecoder { void decodeValue(Inserter inserter) { byte b = in.getByte(); - Cursor cursor = decodeValue(inserter, decode_type(b), decode_meta(b)); + Cursor cursor = decodeValue(inserter, + decode_type(b), + decode_meta(b)); if (!cursor.valid()) { in.fail("failed to decode value"); } diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java index 8f654b46032..4d9fdbba6d4 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java @@ -1,30 +1,28 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.slime; -import static com.yahoo.slime.BinaryFormat.encode_double; -import static com.yahoo.slime.BinaryFormat.encode_type_and_meta; -import static com.yahoo.slime.BinaryFormat.encode_zigzag; +import static com.yahoo.slime.BinaryFormat.*; final class BinaryEncoder implements ArrayTraverser, ObjectSymbolTraverser { - private final BufferedOutput out; + BufferedOutput out; - BinaryEncoder() { - this(new BufferedOutput()); + public BinaryEncoder(int capacity) { + out = new BufferedOutput(capacity); } - BinaryEncoder(BufferedOutput output) { - out = output; + + public BinaryEncoder() { + out = new BufferedOutput(); } - BufferedOutput encode(Slime slime) { + public byte[] encode(Slime slime) { out.reset(); encodeSymbolTable(slime); encodeValue(slime.get()); - return out; + return out.toArray(); } - void encode_cmpr_long(long value) { byte next = (byte)(value & 0x7f); value >>>= 7; // unsigned shift diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java index b6cde18f824..c212500eca7 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.slime; -import com.yahoo.compress.Compressor; - /** * Class for serializing Slime data into binary format, or deserializing * the binary format into a Slime object. @@ -42,17 +40,8 @@ public class BinaryFormat { * @return a new byte array with just the encoded slime. **/ public static byte[] encode(Slime slime) { - return new BinaryEncoder().encode(slime).toArray(); - } - - /** - * Take a Slime object and serialize it into binary format, and compresses it. - * @param slime the object which is to be serialized. - * @param compressor the compressor to use. - * @return a new byte array with just the encoded and compressed slime. - **/ - public static Compressor.Compression encode_and_compress(Slime slime, Compressor compressor) { - return new BinaryEncoder().encode(slime).compress(compressor); + BinaryEncoder encoder = new BinaryEncoder(); + return encoder.encode(slime); } /** diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java index 1e67aac3cb9..8a647269697 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java @@ -19,17 +19,17 @@ final class BufferedInput { position = end; } - BufferedInput(byte[] bytes) { + public BufferedInput(byte[] bytes) { this(bytes, 0, bytes.length); } - BufferedInput(byte[] bytes, int offset, int length) { + public BufferedInput(byte[] bytes, int offset, int length) { this.source = bytes; this.start = offset; position = offset; this.end = offset + length; } - byte getByte() { + public final byte getByte() { if (position == end) { fail("underflow"); return 0; @@ -37,31 +37,31 @@ final class BufferedInput { return source[position++]; } - boolean failed() { + public boolean failed() { return failReason != null; } - boolean eof() { + public boolean eof() { return this.position == this.end; } - String getErrorMessage() { + public String getErrorMessage() { return failReason; } - int getConsumedSize() { + public int getConsumedSize() { return failed() ? 0 : position - start; } - byte[] getOffending() { + public byte[] getOffending() { byte[] ret = new byte[failPos-start]; System.arraycopy(source, start, ret, 0, failPos-start); return ret; } - byte [] getBacking() { return source; } - int getPosition() { return position; } - void skip(int size) { + public final byte [] getBacking() { return source; } + public final int getPosition() { return position; } + public final void skip(int size) { if (position + size > end) { fail("underflow"); } else { @@ -69,7 +69,7 @@ final class BufferedInput { } } - byte[] getBytes(int size) { + public final byte[] getBytes(int size) { if (position + size > end) { fail("underflow"); return new byte[0]; diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java index 61a00185e76..20ed0fc8018 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedOutput.java @@ -1,24 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.slime; -import com.yahoo.compress.Compressor; - -class BufferedOutput { +final class BufferedOutput { private byte[] buf; private int capacity; private int pos; - BufferedOutput(int cap) { + public BufferedOutput(int cap) { capacity = (cap < 64) ? 64 : cap; buf = new byte[capacity]; } - BufferedOutput() { + public BufferedOutput() { this(4096); } - void reset() { + public void reset() { pos = 0; } @@ -33,7 +31,7 @@ class BufferedOutput { } } - int position() { return pos; } + public int position() { return pos; } void put(byte b) { reserve(1); @@ -51,12 +49,9 @@ class BufferedOutput { } } - byte[] toArray() { + public byte[] toArray() { byte[] ret = new byte[pos]; System.arraycopy(buf, 0, ret, 0, pos); return ret; } - Compressor.Compression compress(Compressor compressor) { - return compressor.compress(buf, pos); - } } diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java index cd67e0653dd..7cf4bddaa01 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java @@ -1,42 +1,35 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.slime; -import com.yahoo.compress.CompressionType; -import com.yahoo.compress.Compressor; import org.junit.Test; -import static com.yahoo.slime.BinaryFormat.decode_double; -import static com.yahoo.slime.BinaryFormat.decode_meta; -import static com.yahoo.slime.BinaryFormat.decode_type; -import static com.yahoo.slime.BinaryFormat.decode_zigzag; -import static com.yahoo.slime.BinaryFormat.encode_double; -import static com.yahoo.slime.BinaryFormat.encode_type_and_meta; -import static com.yahoo.slime.BinaryFormat.encode_zigzag; -import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static org.hamcrest.CoreMatchers.*; +import static com.yahoo.slime.BinaryFormat.*; public class BinaryFormatTestCase { static final int TYPE_LIMIT = 8; static final int META_LIMIT = 32; + static final int MAX_CMPR_SIZE = 10; static final int MAX_NUM_SIZE = 8; - static byte enc_t_and_sz(Type t, int size) { + static final byte enc_t_and_sz(Type t, int size) { assert size <= 30; return encode_type_and_meta(t.ID, size + 1); } - static byte enc_t_and_m(Type t, int meta) { + static final byte enc_t_and_m(Type t, int meta) { assert meta <= 31; return encode_type_and_meta(t.ID, meta); } void verify_cmpr_long(long value, byte[] expect) { - BufferedOutput output = new BufferedOutput(); - BinaryEncoder bof = new BinaryEncoder(output); + BinaryEncoder bof = new BinaryEncoder(); bof.encode_cmpr_long(value); - byte[] actual = output.toArray(); + byte[] actual = bof.out.toArray(); assertThat(actual, is(expect)); BinaryDecoder bif = new BinaryDecoder(); @@ -48,10 +41,6 @@ public class BinaryFormatTestCase { // was verifyBasic void verifyEncoding(Slime slime, byte[] expect) { assertThat(BinaryFormat.encode(slime), is(expect)); - Compressor compressor = new Compressor(CompressionType.LZ4, 3, 2, 0); - Compressor.Compression result = BinaryFormat.encode_and_compress(slime, compressor); - byte [] decompressed = compressor.decompress(result); - assertThat(decompressed, is(expect)); verifyMultiEncode(expect); } @@ -234,11 +223,13 @@ public class BinaryFormatTestCase { expect.put(encode_type_and_meta((int)type, (int)(size +1))); } else { expect.put(type); - BinaryEncoder encoder = new BinaryEncoder(expect); + BinaryEncoder encoder = new BinaryEncoder(); + encoder.out = expect; encoder.encode_cmpr_long(size); } { - BinaryEncoder encoder = new BinaryEncoder(actual); + BinaryEncoder encoder = new BinaryEncoder(); + encoder.out = actual; encoder.write_type_and_size(type, size); } assertThat(actual.toArray(), is(expect.toArray())); @@ -282,14 +273,14 @@ public class BinaryFormatTestCase { byte[] expect = expbuf.toArray(); // test output: - BufferedOutput output = new BufferedOutput(); - BinaryEncoder bof = new BinaryEncoder(output); + BinaryEncoder bof = new BinaryEncoder(); + bof.out = new BufferedOutput(); if (hi != 0) { bof.write_type_and_bytes_be(type, bits); } else { bof.write_type_and_bytes_le(type, bits); } - byte[] actual = output.toArray(); + byte[] actual = bof.out.toArray(); assertThat(actual, is(expect)); // test input: @@ -539,8 +530,9 @@ public class BinaryFormatTestCase { 2, enc_t_and_sz(Type.DATA, 4), // f 'd', 'a', 't', 'a' }; + Slime slime = new Slime(); BinaryDecoder decoder = new BinaryDecoder(); - Slime slime = decoder.decode(data); + slime = decoder.decode(data); int consumed = decoder.in.getConsumedSize(); assertThat(consumed, is(data.length)); Cursor c = slime.get(); |