diff options
Diffstat (limited to 'vespajlib/src/test/java/com')
-rw-r--r-- | vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java | 141 | ||||
-rw-r--r-- | vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java | 418 |
2 files changed, 493 insertions, 66 deletions
diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java index 5c3126ce3cf..db001a9276b 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java @@ -31,22 +31,51 @@ public class BinaryFormatTestCase { return encode_type_and_meta(t.ID, meta); } - void verify_cmpr_long(long value, byte[] expect) { + void verify_cmpr_int(int value, byte[] expect) { BufferedOutput output = new BufferedOutput(); BinaryEncoder bof = new BinaryEncoder(output); - bof.encode_cmpr_long(value); + bof.encode_cmpr_int(value); byte[] actual = output.toArray(); assertThat(actual, is(expect)); BinaryDecoder bif = new BinaryDecoder(); bif.in = new BufferedInput(expect); - long got = bif.read_cmpr_long(); + int got = bif.in.read_cmpr_int(); assertThat(got, is(value)); + assertThat(bif.in.failed(), is(false)); + + bif = new BinaryDecoder(); + bif.in = new BufferedInput(expect); + got = bif.in.skip_cmpr_int(); + assertThat(got, is(expect.length - 1)); + assertThat(bif.in.getPosition(), is(expect.length)); + assertThat(bif.in.failed(), is(false)); + + assertThat(BinaryView.peek_cmpr_int_for_testing(expect, 0), is(value)); + assertThat(BinaryView.skip_cmpr_int_for_testing(expect, 0), is(expect.length)); + } + + void verify_read_cmpr_int_fails(byte[] data) { + BinaryDecoder bif = new BinaryDecoder(); + bif.in = new BufferedInput(data); + int got = bif.in.read_cmpr_int(); + assertThat(got, is(0)); + assertThat(bif.in.failed(), is(true)); + + bif = new BinaryDecoder(); + bif.in = new BufferedInput(data); + got = bif.in.skip_cmpr_int(); + assertThat(got, is(data.length - 1)); + assertThat(bif.in.getPosition(), is(data.length)); + assertThat(bif.in.failed(), is(false)); + + assertThat(BinaryView.skip_cmpr_int_for_testing(data, 0), is(data.length)); } // was verifyBasic void verifyEncoding(Slime slime, byte[] expect) { assertThat(BinaryFormat.encode(slime), is(expect)); + assertThat(slime.get().equalTo(BinaryView.inspect(expect)), is(true)); Compressor compressor = new Compressor(CompressionType.LZ4, 3, 2, 0); Compressor.Compression result = BinaryFormat.encode_and_compress(slime, compressor); byte [] decompressed = compressor.decompress(result); @@ -67,7 +96,7 @@ public class BinaryFormatTestCase { @Test public void testZigZagConversion() { - assertThat(encode_zigzag(0), is((long)0)); + assertThat(encode_zigzag(0), is(0L)); assertThat(decode_zigzag(encode_zigzag(0)), is(0L)); assertThat(encode_zigzag(-1), is(1L)); @@ -134,87 +163,59 @@ public class BinaryFormatTestCase { } @Test - public void testCompressedLong() { + public void testCompressedInt() { { - long value = 0; + int value = 0; byte[] wanted = { 0 }; - verify_cmpr_long(value, wanted); + verify_cmpr_int(value, wanted); }{ - long value = 127; + int value = 127; byte[] wanted = { 127 }; - verify_cmpr_long(value, wanted); + verify_cmpr_int(value, wanted); }{ - long value = 128; + int value = 128; byte[] wanted = { -128, 1 }; - verify_cmpr_long(value, wanted); + verify_cmpr_int(value, wanted); }{ - long value = 16383; + int value = 16383; byte[] wanted = { -1, 127 }; - verify_cmpr_long(value, wanted); + verify_cmpr_int(value, wanted); }{ - long value = 16384; + int value = 16384; byte[] wanted = { -128, -128, 1 }; - verify_cmpr_long(value, wanted); + verify_cmpr_int(value, wanted); }{ - long value = 2097151; + int value = 2097151; byte[] wanted = { -1, -1, 127 }; - verify_cmpr_long(value, wanted); + verify_cmpr_int(value, wanted); }{ - long value = 2097152; + int value = 2097152; byte[] wanted = { -128, -128, -128, 1 }; - verify_cmpr_long(value, wanted); + verify_cmpr_int(value, wanted); }{ - long value = 268435455; + int value = 268435455; byte[] wanted = { -1, -1, -1, 127 }; - verify_cmpr_long(value, wanted); + verify_cmpr_int(value, wanted); }{ - long value = 268435456; + int value = 268435456; byte[] wanted = { -128, -128, -128, -128, 1 }; - verify_cmpr_long(value, wanted); - }{ - long value = 34359738367L; - byte[] wanted = { -1, -1, -1, -1, 127 }; - verify_cmpr_long(value, wanted); - }{ - long value = 34359738368L; - byte[] wanted = { -128, -128, -128, -128, -128, 1 }; - verify_cmpr_long(value, wanted); - }{ - long value = 4398046511103L; - byte[] wanted = { -1, -1, -1, -1, -1, 127 }; - verify_cmpr_long(value, wanted); + verify_cmpr_int(value, wanted); }{ - long value = 4398046511104L; - byte[] wanted = { -128, -128, -128, -128, -128, -128, 1 }; - verify_cmpr_long(value, wanted); + int value = 0x7fff_ffff; + byte[] wanted = { -1, -1, -1, -1, 7 }; + verify_cmpr_int(value, wanted); }{ - long value = 562949953421311L; - byte[] wanted = { -1, -1, -1, -1, -1, -1, 127 }; - verify_cmpr_long(value, wanted); + byte[] data = { -1, -1, -1, -1, 8 }; + verify_read_cmpr_int_fails(data); }{ - long value = 562949953421312L; - byte[] wanted = { -128, -128, -128, -128, -128, -128, -128, 1 }; - verify_cmpr_long(value, wanted); + byte[] data = { -1, -1, -1, -1, -1, -1, 1 }; + verify_read_cmpr_int_fails(data); }{ - long value = 72057594037927935L; - byte[] wanted = { -1, -1, -1, -1, -1, -1, -1, 127 }; - verify_cmpr_long(value, wanted); + byte[] data = { -1, -1, -1, -1, -1, -1, -1, -1, 1 }; + verify_read_cmpr_int_fails(data); }{ - long value = 72057594037927936L; - byte[] wanted = { -128, -128, -128, -128, -128, -128, -128, -128, 1 }; - verify_cmpr_long(value, wanted); - }{ - long value = 9223372036854775807L; - byte[] wanted = { -1, -1, -1, -1, -1, -1, -1, -1, 127 }; - verify_cmpr_long(value, wanted); - }{ - long value = -9223372036854775808L; - byte[] wanted = { -128, -128, -128, -128, -128, -128, -128, -128, -128, 1 }; - verify_cmpr_long(value, wanted); - }{ - long value = -1; - byte[] wanted = { -1, -1, -1, -1, -1, -1, -1, -1, -1, 1 }; - verify_cmpr_long(value, wanted); + byte[] data = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1 }; + verify_read_cmpr_int_fails(data); } } @@ -225,16 +226,16 @@ public class BinaryFormatTestCase { @Test public void testTypeAndSizeConversion() { for (byte type = 0; type < TYPE_LIMIT; ++type) { - for (long size = 0; size < 500; ++size) { + for (int size = 0; size < 500; ++size) { BufferedOutput expect = new BufferedOutput(); BufferedOutput actual = new BufferedOutput(); if ((size + 1) < META_LIMIT) { - expect.put(encode_type_and_meta((int)type, (int)(size +1))); + expect.put(encode_type_and_meta(type, size +1)); } else { expect.put(type); BinaryEncoder encoder = new BinaryEncoder(expect); - encoder.encode_cmpr_long(size); + encoder.encode_cmpr_int(size); } { BinaryEncoder encoder = new BinaryEncoder(actual); @@ -247,11 +248,13 @@ public class BinaryFormatTestCase { bif.in = new BufferedInput(got); byte b = bif.in.getByte(); Type decodedType = decode_type(b); - long decodedSize = bif.read_size(decode_meta(b)); + int decodedSize = bif.in.read_size(decode_meta(b)); assertThat(decodedType.ID, is(type)); assertThat(decodedSize, is(size)); assertThat(bif.in.getConsumedSize(), is(got.length)); assertThat(bif.in.failed(), is(false)); + + assertThat(BinaryView.extract_children_for_testing(got, 0), is(size)); } } @@ -299,6 +302,12 @@ public class BinaryFormatTestCase { assertThat(decodedBits, is(bits)); assertThat(bif.in.getConsumedSize(), is(expect.length)); assertThat(bif.in.failed(), is(false)); + + if (hi != 0) { + assertThat(encode_double(BinaryView.extract_double_for_testing(expect, 0)), is(bits)); + } else { + assertThat(encode_zigzag(BinaryView.extract_long_for_testing(expect, 0)), is(bits)); + } } } } diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java new file mode 100644 index 00000000000..568124369d4 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java @@ -0,0 +1,418 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.function.Consumer; + +import org.junit.Test; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static com.yahoo.slime.BinaryFormat.encode_type_and_meta; + +public class BinaryViewTest { + static String makeString(int size) { + var str = new StringBuilder(); + for (int i = 0; i < size; ++i) { + str.append("A"); + } + return str.toString(); + } + static byte[] makeData(int size) { + byte[] data = new byte[size]; + for (int i = 0; i < size; ++i) { + data[i] = 65; + } + return data; + } + static final int numLeafs = 22; + static Cursor insertLeaf(Inserter dst, int id) { + return switch (id) { + case 0 -> dst.insertNIX(); + case 1 -> dst.insertBOOL(false); + case 2 -> dst.insertBOOL(true); + case 3 -> dst.insertLONG(42L); + case 4 -> dst.insertLONG(-42L); + case 5 -> dst.insertLONG(0x1234_5678_8765_4321L); + case 6 -> dst.insertLONG(-0x1234_5678_8765_4321L); + case 7 -> dst.insertDOUBLE(3.5); + case 8 -> dst.insertDOUBLE(1.0/3.0); + case 9 -> dst.insertDOUBLE(-3.5); + case 10 -> dst.insertDOUBLE(-(1.0/3.0)); + case 11 -> dst.insertSTRING(makeString(5)); + case 12 -> dst.insertSTRING(makeString(50)); + case 13 -> dst.insertSTRING(makeString(300)); + case 14 -> dst.insertSTRING(makeData(5)); + case 15 -> dst.insertSTRING(makeData(50)); + case 16 -> dst.insertSTRING(makeData(300)); + case 17 -> dst.insertDATA(makeData(5)); + case 18 -> dst.insertDATA(makeData(50)); + case 19 -> dst.insertDATA(makeData(300)); + case 20 -> dst.insertARRAY(); + case 21 -> dst.insertOBJECT(); + default -> NixValue.invalid(); + }; + } + static Cursor insertInnerObject(Inserter dst) { + var obj = dst.insertOBJECT(); + for (int i = 0; i < numLeafs; ++i) { + assertTrue(insertLeaf(new ObjectInserter(obj, "leaf" + i), i).valid()); + } + return obj; + } + static Cursor insertInnerArray(Inserter dst) { + var arr = dst.insertARRAY(); + for (int i = 0; i < numLeafs; ++i) { + assertTrue(insertLeaf(new ArrayInserter(arr), i).valid()); + } + return arr; + } + static Cursor insertOuterObject(Inserter dst) { + var obj = dst.insertOBJECT(); + assertTrue(insertInnerObject(new ObjectInserter(obj, "foo")).valid()); + assertTrue(insertInnerArray(new ObjectInserter(obj, "bar")).valid()); + return obj; + } + static Cursor insertOuterArray(Inserter dst) { + var arr = dst.insertARRAY(); + assertTrue(insertInnerObject(new ArrayInserter(arr)).valid()); + assertTrue(insertInnerArray(new ArrayInserter(arr)).valid()); + return arr; + } + static Cursor insertManySymbols(Inserter dst) { + var obj = dst.insertOBJECT(); + for (int i = 0; i < 300; ++i) { + obj.setLong("val" + i, i); + } + assertEquals(300, obj.fields()); + return obj; + } + static Cursor insertLargeArray(Inserter dst) { + var arr = dst.insertARRAY(); + for (int i = 0; i < 300; ++i) { + arr.addLong(i); + } + assertEquals(300, arr.entries()); + return arr; + } + static final int numShapes = numLeafs + 6; + static Cursor insertRoot(Slime dst, int shape) { + var root = new SlimeInserter(dst); + if (shape < numLeafs) { + return insertLeaf(root, shape); + } + return switch (shape) { + case (numLeafs) -> insertInnerObject(root); + case (numLeafs + 1) -> insertInnerArray(root); + case (numLeafs + 2) -> insertOuterObject(root); + case (numLeafs + 3) -> insertOuterArray(root); + case (numLeafs + 4) -> insertManySymbols(root); + case (numLeafs + 5) -> insertLargeArray(root); + default -> NixValue.invalid(); + }; + } + static Slime makeSlime(int shape) { + var slime = new Slime(); + var root = insertRoot(slime, shape); + assertTrue(root.valid()); + return slime; + } + + class MyConsumer implements Consumer<Inspector> { + Inspector value = null; + @Override public void accept(Inspector value) { + assertNull(ctx, this.value); + this.value = value; + } + }; + + void checkConsumer(Inspector view) { + var consumer = new MyConsumer(); + view.ifValid(consumer); + assertEquals(ctx, view.valid(), consumer.value != null); + if (view.valid()) { + assertSame(ctx, view, consumer.value); + } + } + + class MyVisitor implements Visitor { + enum Called { NONE, INVALID, NIX, BOOL, LONG, DOUBLE, UTF8, DATA, ARRAY, OBJECT } + Called called = Called.NONE; + boolean boolValue; + long longValue; + double doubleValue; + byte[] bytes; + Inspector stuff; + @Override public void visitInvalid() { + assertEquals(ctx, Called.NONE, called); + called = Called.INVALID; + } + @Override public void visitNix() { + assertEquals(ctx, Called.NONE, called); + called = Called.NIX; + } + @Override public void visitBool(boolean bit) { + assertEquals(ctx, Called.NONE, called); + called = Called.BOOL; + boolValue = bit; + } + @Override public void visitLong(long l) { + assertEquals(ctx, Called.NONE, called); + called = Called.LONG; + longValue = l; + } + @Override public void visitDouble(double d) { + assertEquals(ctx, Called.NONE, called); + called = Called.DOUBLE; + doubleValue = d; + } + @Override public void visitString(String str) { + fail(ctx + ", strings are never utf-16 in binary view"); + } + @Override public void visitString(byte[] utf8) { + assertEquals(ctx, Called.NONE, called); + called = Called.UTF8; + bytes = utf8; + } + @Override public void visitData(byte[] data) { + assertEquals(ctx, Called.NONE, called); + called = Called.DATA; + bytes = data; + } + @Override public void visitArray(Inspector arr) { + assertEquals(ctx, Called.NONE, called); + called = Called.ARRAY; + stuff = arr; + } + @Override public void visitObject(Inspector obj) { + assertEquals(ctx, Called.NONE, called); + called = Called.OBJECT; + stuff = obj; + } + }; + + void checkVisitor(Inspector view) { + var visitor = new MyVisitor(); + view.accept(visitor); + if (!view.valid()) { + assertEquals(ctx, MyVisitor.Called.INVALID, visitor.called); + return; + } + switch (view.type()) { + case NIX: + assertEquals(ctx, MyVisitor.Called.NIX, visitor.called); + break; + case BOOL: + assertEquals(ctx, MyVisitor.Called.BOOL, visitor.called); + assertEquals(ctx, view.asBool(), visitor.boolValue); + break; + case LONG: + assertEquals(ctx, MyVisitor.Called.LONG, visitor.called); + assertEquals(ctx, view.asLong(), visitor.longValue); + break; + case DOUBLE: + assertEquals(ctx, MyVisitor.Called.DOUBLE, visitor.called); + assertEquals(ctx, view.asDouble(), visitor.doubleValue, 0.0); + break; + case STRING: + assertEquals(ctx, MyVisitor.Called.UTF8, visitor.called); + assertArrayEquals(ctx, view.asUtf8(), visitor.bytes); + break; + case DATA: + assertEquals(ctx, MyVisitor.Called.DATA, visitor.called); + assertArrayEquals(ctx, view.asData(), visitor.bytes); + break; + case ARRAY: + assertEquals(ctx, MyVisitor.Called.ARRAY, visitor.called); + assertSame(ctx, view, visitor.stuff); + break; + case OBJECT: + assertEquals(ctx, MyVisitor.Called.OBJECT, visitor.called); + assertSame(ctx, view, visitor.stuff); + break; + default: + fail(ctx + ", should not be reached"); + break; + } + } + + class MyArrayTraverser implements ArrayTraverser { + ArrayList<Inspector> list = new ArrayList<>(); + @Override public void entry(int idx, Inspector value) { + list.add(value); + } + } + + void checkTraverseArray(Inspector value, Inspector view) { + var a = new MyArrayTraverser(); + var b = new MyArrayTraverser(); + value.traverse(a); + view.traverse(b); + assertEquals(ctx, a.list.size(), b.list.size()); + for (int i = 0; i < a.list.size(); ++i) { + checkParity(a.list.get(i), b.list.get(i)); + } + } + + class MyObjectSymbolTraverser implements ObjectSymbolTraverser { + HashMap<Integer,Inspector> map = new HashMap<>(); + @Override public void field(int sym, Inspector value) { + map.put(sym, value); + } + } + + void checkTraverseObjectSymbol(Inspector value, Inspector view) { + var a = new MyObjectSymbolTraverser(); + var b = new MyObjectSymbolTraverser(); + value.traverse(a); + view.traverse(b); + assertEquals(ctx, a.map.size(), b.map.size()); + for (Integer key: a.map.keySet()) { + assertTrue(ctx, b.map.containsKey(key)); + checkParity(a.map.get(key), b.map.get(key)); + } + } + + class MyObjectTraverser implements ObjectTraverser { + HashMap<String,Inspector> map = new HashMap<>(); + @Override public void field(String name, Inspector value) { + map.put(name, value); + } + } + + void checkTraverseObject(Inspector value, Inspector view) { + var a = new MyObjectTraverser(); + var b = new MyObjectTraverser(); + value.traverse(a); + view.traverse(b); + assertEquals(ctx, a.map.size(), b.map.size()); + for (String key: a.map.keySet()) { + assertTrue(ctx, b.map.containsKey(key)); + checkParity(a.map.get(key), b.map.get(key)); + } + } + void checkParity(Inspector value, Inspector view) { + checkConsumer(view); + checkVisitor(view); + if (value == view) { + // avoid infinite invalid nix recursion + assertSame(ctx, value, view); + return; + } + assertEquals(ctx, value.valid(), view.valid()); + assertEquals(ctx, value.type(), view.type()); + assertEquals(ctx, value.children(), view.children()); + assertEquals(ctx, value.entries(), view.entries()); + assertEquals(ctx, value.fields(), view.fields()); + assertEquals(ctx, value.asBool(), view.asBool()); + assertEquals(ctx, value.asLong(), view.asLong()); + assertEquals(ctx, value.asDouble(), view.asDouble(), 0.0); + assertEquals(ctx, value.asString(), view.asString()); + assertArrayEquals(ctx, value.asUtf8(), view.asUtf8()); + assertArrayEquals(ctx, value.asData(), view.asData()); + checkTraverseArray(value, view); + checkTraverseObjectSymbol(value, view); + checkTraverseObject(value, view); + checkParity(value.entry(0), view.entry(0)); + checkParity(value.entry(1), view.entry(1)); + checkParity(value.entry(2), view.entry(2)); + checkParity(value.entry(3), view.entry(3)); + checkParity(value.entry(200), view.entry(200)); + checkParity(value.entry(500), view.entry(500)); + checkParity(value.entry(-1), view.entry(-1)); + checkParity(value.field(0), view.field(0)); + checkParity(value.field(1), view.field(1)); + checkParity(value.field(2), view.field(2)); + checkParity(value.field(3), view.field(3)); + checkParity(value.field(SymbolTable.INVALID), view.field(SymbolTable.INVALID)); + checkParity(value.field(-1), view.field(-1)); + checkParity(value.field("foo"), view.field("foo")); + checkParity(value.field("bar"), view.field("bar")); + checkParity(value.field("val256"), view.field("val256")); + checkParity(value.field("bogus"), view.field("bogus")); + assertTrue(ctx, value.equalTo(view)); + assertTrue(ctx, view.equalTo(value)); + } + + String ctx; + @Test public void testBinaryViewShapesParity() { + for (int i = 0; i < numShapes; ++i) { + var slime = makeSlime(i); + ctx = "case " + i + ": '" + slime.toString() + "'"; + byte[] data = BinaryFormat.encode(slime); + try { + checkParity(slime.get(), BinaryView.inspect(data)); + } catch (Exception e) { + fail(ctx + ", got exception: " + e); + } + } + } + + @Test public void testTrivialView() { + byte[] data = {0, 0}; + var input = new BufferedInput(data); + var view = BinaryView.inspectImpl(input); + assertTrue(view.valid()); + assertEquals(Type.NIX, view.type()); + assertFalse(input.failed()); + } + + @Test public void testUnderflow() { + byte[] data = {}; + var input = new BufferedInput(data); + var view = BinaryView.inspectImpl(input); + assertFalse(view.valid()); + assertTrue(input.failed()); + assertEquals("underflow", input.getErrorMessage()); + } + + @Test public void testMultiByteUnderflow() { + byte[] data = { 0, encode_type_and_meta(Type.STRING.ID, 3), 65 }; + var input = new BufferedInput(data); + var view = BinaryView.inspectImpl(input); + assertFalse(view.valid()); + assertTrue(input.failed()); + assertEquals("underflow", input.getErrorMessage()); + } + + @Test public void testCompressedIntOverflow() { + byte[] data = { -1, -1, -1, -1, 8 }; + var input = new BufferedInput(data); + var view = BinaryView.inspectImpl(input); + assertFalse(view.valid()); + assertTrue(input.failed()); + assertEquals("compressed int overflow", input.getErrorMessage()); + } + + @Test public void testExtBitsOverflow() { + byte[] data = { 0, encode_type_and_meta(Type.OBJECT.ID, 2), -1, -1, -1, -1, 1 }; + var input = new BufferedInput(data); + var view = BinaryView.inspectImpl(input); + assertFalse(view.valid()); + assertTrue(input.failed()); + assertEquals("symbol id too big", input.getErrorMessage()); + } + + @Test public void testDecodeIndexOverflowArray() { + byte[] data = { 0, encode_type_and_meta(Type.ARRAY.ID, 4) }; + var input = new BufferedInput(data); + var view = BinaryView.inspectImpl(input); + assertFalse(view.valid()); + assertTrue(input.failed()); + assertEquals("decode index too big", input.getErrorMessage()); + } + + @Test public void testDecodeIndexOverflowObject() { + byte[] data = { 0, encode_type_and_meta(Type.OBJECT.ID, 4) }; + var input = new BufferedInput(data); + var view = BinaryView.inspectImpl(input); + assertFalse(view.valid()); + assertTrue(input.failed()); + assertEquals("decode index too big", input.getErrorMessage()); + } +} |