BinaryView; inspect slime value in binary format

Use int instead of long for stand-alone compressed values (sizes and
symbol ids). Also added overflow/wrap-around checks for these values
to avoid things like infinite recursion due to negative buffer skips
during DecodeIndex creation. This makes decoding fail in more
deterministic ways and also aligns with Java using int for sizes.

8 files changed, 856 insertions, 113 deletions

diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java
index b9aa8e5cf22..af6a2ae80a3 100644
--- a/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java
@@ -23,7 +23,7 @@ final class BinaryDecoder {
     public Slime decode(byte[] bytes, int offset, int length) {
         Slime slime = new Slime();
         in = new BufferedInput(bytes, offset, length);
-        decodeSymbolTable(slime);
+        decodeSymbolTable(in, slime.symbolTable());
         decodeValue(slimeInserter.adjust(slime));
         if (in.failed()) {
             slime.wrap("partial_result");
@@ -33,22 +33,6 @@ final class BinaryDecoder {
         return slime;
     }
 
-    long read_cmpr_long() {
-        long next = in.getByte();
-        long value = (next & 0x7f);
-        int shift = 7;
-        while ((next & 0x80) != 0) {
-            next = in.getByte();
-            value |= ((next & 0x7f) << shift);
-            shift += 7;
-        }
-        return value;
-    }
-
-    long read_size(int meta) {
-        return (meta == 0) ? read_cmpr_long() : (meta - 1);
-    }
-
     long read_bytes_le(int bytes) {
         long value = 0;
         int shift = 0;
@@ -90,22 +74,20 @@ final class BinaryDecoder {
     }
 
     Cursor decodeSTRING(Inserter inserter, int meta) {
-        long size = read_size(meta);
-        int sz = (int)size; // XXX
-        byte[] image = in.getBytes(sz);
+        int size = in.read_size(meta);
+        byte[] image = in.getBytes(size);
         return inserter.insertSTRING(image);
     }
 
     Cursor decodeDATA(Inserter inserter, int meta) {
-        long size = read_size(meta);
-        int sz = (int)size; // XXX
-        byte[] image = in.getBytes(sz);
+        int size = in.read_size(meta);
+        byte[] image = in.getBytes(size);
         return inserter.insertDATA(image);
     }
 
     Cursor decodeARRAY(Inserter inserter, int meta) {
         Cursor cursor = inserter.insertARRAY();
-        long size = read_size(meta);
+        int size = in.read_size(meta);
         for (int i = 0; i < size; ++i) {
             decodeValue(arrayInserter.adjust(cursor));
         }
@@ -114,10 +96,9 @@ final class BinaryDecoder {
 
     Cursor decodeOBJECT(Inserter inserter, int meta) {
         Cursor cursor = inserter.insertOBJECT();
-        long size = read_size(meta);
+        int size = in.read_size(meta);
         for (int i = 0; i < size; ++i) {
-            long l = read_cmpr_long();
-            int symbol = (int)l; // check for overflow?
+            int symbol = in.read_cmpr_int();
             decodeValue(objectInserter.adjust(cursor, symbol));
         }
         return cursor;
@@ -146,20 +127,18 @@ final class BinaryDecoder {
         }
     }
 
-    void decodeSymbolTable(Slime slime) {
-        long numSymbols = read_cmpr_long();
-        final byte [] backing = in.getBacking();
+    static void decodeSymbolTable(BufferedInput input, SymbolTable names) {
+        int numSymbols = input.read_cmpr_int();
+        final byte[] backing = input.getBacking();
         for (int i = 0; i < numSymbols; ++i) {
-            long size = read_cmpr_long();
-            int sz = (int)size; // XXX
-            int offset = in.getPosition();
-            in.skip(sz);
-            int symbol = slime.insert(Utf8Codec.decode(backing, offset, sz));
+            int size = input.read_cmpr_int();
+            int offset = input.getPosition();
+            input.skip(size);
+            int symbol = names.insert(Utf8Codec.decode(backing, offset, size));
             if (symbol != i) {
-                in.fail("duplicate symbols in symbol table");
+                input.fail("duplicate symbols in symbol table");
                 return;
             }
         }
     }
-
 }
diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java
index 7da85b5cb63..f12496f7a76 100644
--- a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java
@@ -24,7 +24,7 @@ final class BinaryEncoder implements ArrayTraverser, ObjectSymbolTraverser {
     }
 
 
-    void encode_cmpr_long(long value) {
+    void encode_cmpr_int(int value) {
         byte next = (byte)(value & 0x7f);
         value >>>= 7; // unsigned shift
         while (value != 0) {
@@ -36,12 +36,12 @@ final class BinaryEncoder implements ArrayTraverser, ObjectSymbolTraverser {
         out.put(next);
     }
 
-    void write_type_and_size(int type, long size) {
+    void write_type_and_size(int type, int size) {
         if (size <= 30) {
-            out.put(encode_type_and_meta(type, (int)(size + 1)));
+            out.put(encode_type_and_meta(type, size + 1));
         } else {
             out.put(encode_type_and_meta(type, 0));
-            encode_cmpr_long(size);
+            encode_cmpr_int(size);
         }
     }
 
@@ -125,11 +125,11 @@ final class BinaryEncoder implements ArrayTraverser, ObjectSymbolTraverser {
 
     void encodeSymbolTable(Slime slime) {
         int numSymbols = slime.symbols();
-        encode_cmpr_long(numSymbols);
+        encode_cmpr_int(numSymbols);
         for (int i = 0 ; i < numSymbols; ++i) {
             String name = slime.inspect(i);
             byte[] bytes = Utf8Codec.encode(name);
-            encode_cmpr_long(bytes.length);
+            encode_cmpr_int(bytes.length);
             out.put(bytes);
         }
     }
@@ -139,7 +139,7 @@ final class BinaryEncoder implements ArrayTraverser, ObjectSymbolTraverser {
     }
 
     public void field(int symbol, Inspector inspector) {
-        encode_cmpr_long(symbol);
+        encode_cmpr_int(symbol);
         encodeValue(inspector);
     }
 
diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryView.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryView.java
new file mode 100644
index 00000000000..0e111d42061
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryView.java
@@ -0,0 +1,307 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.slime;
+
+import java.util.function.Consumer;
+import static com.yahoo.slime.BinaryFormat.decode_double;
+import static com.yahoo.slime.BinaryFormat.decode_meta;
+import static com.yahoo.slime.BinaryFormat.decode_type;
+import static com.yahoo.slime.BinaryFormat.decode_zigzag;
+
+/**
+ * A read-only view of a Slime value that is stored in binary format.
+ **/
+public final class BinaryView implements Inspector {
+
+    private final byte[] data;
+    private final SymbolTable names;
+    private final DecodeIndex index;
+    private final int self;
+
+    private BinaryView(byte[] data, SymbolTable names, DecodeIndex index, int self) {
+        this.data = data;
+        this.names = names;
+        this.index = index;
+        this.self = self;
+    }
+    private int peek_cmpr_int(int idx) {
+        long next = data[idx++];
+        long value = (next & 0x7f);
+        int shift = 7;
+        while ((next & 0x80) != 0) {
+            next = data[idx++];
+            value |= ((next & 0x7f) << shift);
+            shift += 7;
+        }
+        return (int)value;
+    }
+    private int skip_cmpr_int(int idx) {
+        while ((data[idx++] & 0x80) != 0);
+        return idx;
+    }
+    private int extract_children(int idx) {
+        int bytes = decode_meta(data[idx++]);
+        return (bytes == 0)
+                ? peek_cmpr_int(idx)
+                : (bytes - 1);
+    }
+    private long extract_long(int idx) {
+        int bytes = decode_meta(data[idx++]);
+        long value = 0;
+        int shift = 0;
+        for (int i = 0; i < bytes; ++i) {
+            long b = data[idx++];
+            value |= (b & 0xff) << shift;
+            shift += 8;
+        }
+        return decode_zigzag(value);
+    }
+    private double extract_double(int idx) {
+        int bytes = decode_meta(data[idx++]);
+        long value = 0;
+        int shift = 56;
+        for (int i = 0; i < bytes; ++i) {
+            long b = data[idx++];
+            value |= (b & 0xff) << shift;
+            shift -= 8;
+        }
+        return decode_double(value);
+    }
+    private String extract_string(int idx) {
+        int bytes = decode_meta(data[idx++]);
+        if (bytes == 0) {
+            bytes = peek_cmpr_int(idx);
+            idx = skip_cmpr_int(idx);
+        } else {
+            --bytes;
+        }
+        return Utf8Codec.decode(data, idx, bytes);
+    }
+    private byte[] extract_bytes(int idx) {
+        int bytes = decode_meta(data[idx++]);
+        if (bytes == 0) {
+            bytes = peek_cmpr_int(idx);
+            idx = skip_cmpr_int(idx);
+        } else {
+            --bytes;
+        }
+        byte[] ret = new byte[bytes];
+        for (int i = 0; i < bytes; ++i) {
+            ret[i] = data[idx++];
+        }
+        return ret;
+    }
+    private Inspector find_field(int pos, int len, int sym) {
+        for (int i = 0; i < len; ++i) {
+            int idx = index.getByteOffset(pos + i);
+            if (peek_cmpr_int(idx - (index.getExtBits(pos + i) + 1)) == sym) {
+                return new BinaryView(data, names, index, pos + i);
+            }
+        }
+        return NixValue.invalid();
+    }
+
+    @Override public boolean valid() { return true; }
+    @Override public void ifValid(Consumer<Inspector> consumer) { consumer.accept(this); }
+    @Override public Type type() { return decode_type(data[index.getByteOffset(self)]); }
+    @Override public int children() {
+        return switch (type()) {
+            case OBJECT, ARRAY -> extract_children(index.getByteOffset(self));
+            default -> 0;
+        };
+    }
+    @Override public int entries() {
+        return switch (type()) {
+            case ARRAY -> extract_children(index.getByteOffset(self));
+            default -> 0;
+        };
+    }
+    @Override public int fields() {
+        return switch (type()) {
+            case OBJECT -> extract_children(index.getByteOffset(self));
+            default -> 0;
+        };
+    }
+    @Override public boolean asBool() {
+        return switch (type()) {
+            case BOOL -> (decode_meta(data[index.getByteOffset(self)]) != 0);
+            default -> false;
+        };
+    }
+    @Override public long asLong() {
+        return switch (type()) {
+            case LONG -> extract_long(index.getByteOffset(self));
+            case DOUBLE -> (long)extract_double(index.getByteOffset(self));
+            default -> 0;
+        };
+    }
+    @Override public double asDouble() {
+        return switch (type()) {
+            case LONG -> extract_long(index.getByteOffset(self));
+            case DOUBLE -> extract_double(index.getByteOffset(self));
+            default -> 0.0;
+        };
+    }
+    @Override public String asString() {
+        return switch (type()) {
+            case STRING -> extract_string(index.getByteOffset(self));
+            default -> Value.emptyString;
+        };
+    }
+    @Override public byte[] asUtf8() {
+        return switch (type()) {
+            case STRING -> extract_bytes(index.getByteOffset(self));
+            default -> Value.emptyData;
+        };
+    }
+    @Override public byte[] asData() {
+        return switch (type()) {
+            case DATA -> extract_bytes(index.getByteOffset(self));
+            default -> Value.emptyData;
+        };
+    }
+    @Override public void accept(Visitor v) {
+        switch (type()) {
+            case NIX:    v.visitNix(); break;
+            case BOOL:   v.visitBool(decode_meta(data[index.getByteOffset(self)]) != 0); break;
+            case LONG:   v.visitLong(extract_long(index.getByteOffset(self))); break;
+            case DOUBLE: v.visitDouble(extract_double(index.getByteOffset(self))); break;
+            case STRING: v.visitString(extract_bytes(index.getByteOffset(self))); break;
+            case DATA:   v.visitData(extract_bytes(index.getByteOffset(self))); break;
+            case ARRAY:  v.visitArray(this); break;
+            case OBJECT: v.visitObject(this); break;
+            default: throw new RuntimeException("should not be reached");
+        }
+    }
+    @Override public void traverse(ArrayTraverser at) {
+        int pos = index.getFirstChild(self);
+        int len = entries();
+        for (int i = 0; i < len; ++i) {
+            at.entry(i, new BinaryView(data, names, index, pos + i));
+        }
+    }
+    @Override public void traverse(ObjectSymbolTraverser ot) {
+        int pos = index.getFirstChild(self);
+        int len = fields();
+        for (int i = 0; i < len; ++i) {
+            int sym = peek_cmpr_int(index.getByteOffset(pos + i) - (index.getExtBits(pos + i) + 1));
+            ot.field(sym, new BinaryView(data, names, index, pos + i));
+        }
+    }
+    @Override public void traverse(ObjectTraverser ot) {
+        int pos = index.getFirstChild(self);
+        int len = fields();
+        for (int i = 0; i < len; ++i) {
+            int sym = peek_cmpr_int(index.getByteOffset(pos + i) - (index.getExtBits(pos + i) + 1));
+            ot.field(names.inspect(sym), new BinaryView(data, names, index, pos + i));
+        }
+    }
+    @Override public Inspector entry(int idx) {
+        int limit = entries();
+        if (idx >= 0 && idx < limit) {
+            return new BinaryView(data, names, index, index.getFirstChild(self) + idx);
+        }
+        return NixValue.invalid();
+    }
+    @Override public Inspector field(int sym) {
+        int limit = fields();
+        if (limit > 0 && sym != SymbolTable.INVALID) {
+            return find_field(index.getFirstChild(self), limit, sym);
+        }
+        return NixValue.invalid();
+    }
+    @Override public Inspector field(String name) {
+        int limit = fields();
+        if (limit > 0) {
+            int sym = names.lookup(name);
+            if (sym != SymbolTable.INVALID) {
+                return find_field(index.getFirstChild(self), limit, sym);
+            }
+        }
+        return NixValue.invalid();
+    }
+
+    private static void buildIndex(BufferedInput input, DecodeIndex index, int self, int extBits) {
+        int pos = input.getPosition();
+        byte tag = input.getByte();
+        Type type = decode_type(tag);
+        int meta = decode_meta(tag);
+        switch (type) {
+            case NIX:
+            case BOOL:
+                index.set(self, pos, 0, extBits);
+                break;
+            case LONG:
+            case DOUBLE:
+                input.skip(meta);
+                index.set(self, pos, 0, extBits);
+                break;
+            case STRING:
+            case DATA: {
+                int size = input.read_size(meta);
+                input.skip(size);
+                index.set(self, pos, 0, extBits);
+                break; }
+            case ARRAY: {
+                int size = input.read_size(meta);
+                if (size > input.getBacking().length - index.size()) {
+                    input.fail("decode index too big");
+                    return;
+                }
+                int firstChild = index.reserve(size);
+                index.set(self, pos, firstChild, extBits);
+                for (int i = 0; i < size; ++i) {
+                    buildIndex(input, index, firstChild + i, 0);
+                }
+                break; }
+            case OBJECT: {
+                int size = input.read_size(meta);
+                if (size > input.getBacking().length - index.size()) {
+                    input.fail("decode index too big");
+                    return;
+                }
+                int firstChild = index.reserve(size);
+                index.set(self, pos, firstChild, extBits);
+                for (int i = 0; i < size; ++i) {
+                    int childExtBits = input.skip_cmpr_int();
+                    if (childExtBits > 3) {
+                        input.fail("symbol id too big");
+                        return;
+                    }
+                    buildIndex(input, index, firstChild + i, childExtBits);
+                }
+                break; }
+            default: throw new RuntimeException("should not be reached");
+        }
+    }
+
+    static Inspector inspectImpl(BufferedInput input) {
+        var names = new SymbolTable();
+        var index = new DecodeIndex();
+        BinaryDecoder.decodeSymbolTable(input, names);
+        buildIndex(input, index, index.reserve(1), 0);
+        if (input.failed()) {
+            return NixValue.invalid();
+        }
+        return new BinaryView(input.getBacking(), names, index, 0);
+    }
+
+    public static Inspector inspect(byte[] data) {
+        return inspectImpl(new BufferedInput(data));
+    }
+
+    static int peek_cmpr_int_for_testing(byte[] data, int idx) {
+        return new BinaryView(data, null, null, -1).peek_cmpr_int(idx);
+    }
+    static int skip_cmpr_int_for_testing(byte[] data, int idx) {
+        return new BinaryView(data, null, null, -1).skip_cmpr_int(idx);
+    }
+    static int extract_children_for_testing(byte[] data, int idx) {
+        return new BinaryView(data, null, null, -1).extract_children(idx);
+    }
+    static long extract_long_for_testing(byte[] data, int idx) {
+        return new BinaryView(data, null, null, -1).extract_long(idx);
+    }
+    static double extract_double_for_testing(byte[] data, int idx) {
+        return new BinaryView(data, null, null, -1).extract_double(idx);
+    }
+}
diff --git a/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java b/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java
index ddbb25196b5..5c994d7f793 100644
--- a/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/BufferedInput.java
@@ -59,7 +59,7 @@ final class BufferedInput {
         return ret;
     }
 
-    byte [] getBacking() { return source; }
+    byte[] getBacking() { return source; }
     int getPosition() { return position; }
     void skip(int size) {
         if (position + size > end) {
@@ -80,4 +80,32 @@ final class BufferedInput {
         }
         return ret;
     }
+
+    int read_cmpr_int() {
+        long next = getByte();
+        long value = (next & 0x7f);
+        int shift = 7;
+        while (shift < 32 && (next & 0x80) != 0) {
+            next = getByte();
+            value |= ((next & 0x7f) << shift);
+            shift += 7;
+        }
+        if (value > 0x7fff_ffffL) {
+            fail("compressed int overflow");
+            value = 0;
+        }
+        return (int)value;
+    }
+
+    int skip_cmpr_int() {
+        int extBits = 0;
+        while ((getByte() & 0x80) != 0) {
+            ++extBits;
+        }
+        return extBits;
+    }
+
+    int read_size(int meta) {
+        return (meta == 0) ? read_cmpr_int() : (meta - 1);
+    }
 }
diff --git a/vespajlib/src/main/java/com/yahoo/slime/Slime.java b/vespajlib/src/main/java/com/yahoo/slime/Slime.java
index eba9226c8ef..7d29131cbdb 100644
--- a/vespajlib/src/main/java/com/yahoo/slime/Slime.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/Slime.java
@@ -13,6 +13,8 @@ public final class Slime {
     private final SymbolTable names = new SymbolTable();
     private Value root = NixValue.instance();
 
+    SymbolTable symbolTable() { return names; }
+
     /**
      * Construct an empty Slime with an empty top-level value.
      */
diff --git a/vespajlib/src/main/java/com/yahoo/slime/Value.java b/vespajlib/src/main/java/com/yahoo/slime/Value.java
index 1943e77663f..6a1d7b2dd8e 100644
--- a/vespajlib/src/main/java/com/yahoo/slime/Value.java
+++ b/vespajlib/src/main/java/com/yahoo/slime/Value.java
@@ -13,8 +13,8 @@ import java.util.function.Consumer;
  */
 abstract class Value implements Cursor {
 
-    private static final String emptyString = "";
-    private static final byte[] emptyData = new byte[0];
+    static final String emptyString = "";
+    static final byte[] emptyData = new byte[0];
 
     public final boolean valid() { return this != NixValue.invalid(); }
 
diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java
index 5c3126ce3cf..db001a9276b 100644
--- a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java
@@ -31,22 +31,51 @@ public class BinaryFormatTestCase {
         return encode_type_and_meta(t.ID, meta);
     }
 
-    void verify_cmpr_long(long value, byte[] expect) {
+    void verify_cmpr_int(int value, byte[] expect) {
         BufferedOutput output = new BufferedOutput();
         BinaryEncoder bof = new BinaryEncoder(output);
-        bof.encode_cmpr_long(value);
+        bof.encode_cmpr_int(value);
         byte[] actual = output.toArray();
         assertThat(actual, is(expect));
 
         BinaryDecoder bif = new BinaryDecoder();
         bif.in = new BufferedInput(expect);
-        long got = bif.read_cmpr_long();
+        int got = bif.in.read_cmpr_int();
         assertThat(got, is(value));
+        assertThat(bif.in.failed(), is(false));
+
+        bif = new BinaryDecoder();
+        bif.in = new BufferedInput(expect);
+        got = bif.in.skip_cmpr_int();
+        assertThat(got, is(expect.length - 1));
+        assertThat(bif.in.getPosition(), is(expect.length));
+        assertThat(bif.in.failed(), is(false));
+
+        assertThat(BinaryView.peek_cmpr_int_for_testing(expect, 0), is(value));
+        assertThat(BinaryView.skip_cmpr_int_for_testing(expect, 0), is(expect.length));
+    }
+
+    void verify_read_cmpr_int_fails(byte[] data) {
+        BinaryDecoder bif = new BinaryDecoder();
+        bif.in = new BufferedInput(data);
+        int got = bif.in.read_cmpr_int();
+        assertThat(got, is(0));
+        assertThat(bif.in.failed(), is(true));
+
+        bif = new BinaryDecoder();
+        bif.in = new BufferedInput(data);
+        got = bif.in.skip_cmpr_int();
+        assertThat(got, is(data.length - 1));
+        assertThat(bif.in.getPosition(), is(data.length));
+        assertThat(bif.in.failed(), is(false));
+
+        assertThat(BinaryView.skip_cmpr_int_for_testing(data, 0), is(data.length));
     }
 
     // was verifyBasic
     void verifyEncoding(Slime slime, byte[] expect) {
         assertThat(BinaryFormat.encode(slime), is(expect));
+        assertThat(slime.get().equalTo(BinaryView.inspect(expect)), is(true));
         Compressor compressor = new Compressor(CompressionType.LZ4, 3, 2, 0);
         Compressor.Compression result = BinaryFormat.encode_and_compress(slime, compressor);
         byte [] decompressed = compressor.decompress(result);
@@ -67,7 +96,7 @@ public class BinaryFormatTestCase {
 
     @Test
     public void testZigZagConversion() {
-        assertThat(encode_zigzag(0), is((long)0));
+        assertThat(encode_zigzag(0), is(0L));
         assertThat(decode_zigzag(encode_zigzag(0)), is(0L));
 
         assertThat(encode_zigzag(-1), is(1L));
@@ -134,87 +163,59 @@ public class BinaryFormatTestCase {
     }
 
     @Test
-    public void testCompressedLong() {
+    public void testCompressedInt() {
         {
-            long value = 0;
+            int value = 0;
             byte[] wanted = { 0 };
-            verify_cmpr_long(value, wanted);
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 127;
+            int value = 127;
             byte[] wanted = { 127 };
-            verify_cmpr_long(value, wanted);
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 128;
+            int value = 128;
             byte[] wanted = { -128, 1 };
-            verify_cmpr_long(value, wanted);
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 16383;
+            int value = 16383;
             byte[] wanted = { -1, 127 };
-            verify_cmpr_long(value, wanted);
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 16384;
+            int value = 16384;
             byte[] wanted = { -128, -128, 1 };
-            verify_cmpr_long(value, wanted);
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 2097151;
+            int value = 2097151;
             byte[] wanted = { -1, -1, 127 };
-            verify_cmpr_long(value, wanted);
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 2097152;
+            int value = 2097152;
             byte[] wanted = { -128, -128, -128, 1 };
-            verify_cmpr_long(value, wanted);
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 268435455;
+            int value = 268435455;
             byte[] wanted = { -1, -1, -1, 127 };
-            verify_cmpr_long(value, wanted);
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 268435456;
+            int value = 268435456;
             byte[] wanted = { -128, -128, -128, -128, 1 };
-            verify_cmpr_long(value, wanted);
-        }{
-            long value = 34359738367L;
-            byte[] wanted = { -1, -1, -1, -1, 127 };
-            verify_cmpr_long(value, wanted);
-        }{
-            long value = 34359738368L;
-            byte[] wanted = { -128, -128, -128, -128, -128, 1 };
-            verify_cmpr_long(value, wanted);
-        }{
-            long value = 4398046511103L;
-            byte[] wanted = { -1, -1, -1, -1, -1, 127 };
-            verify_cmpr_long(value, wanted);
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 4398046511104L;
-            byte[] wanted = { -128, -128, -128, -128, -128, -128, 1 };
-            verify_cmpr_long(value, wanted);
+            int value = 0x7fff_ffff;
+            byte[] wanted = { -1, -1, -1, -1, 7 };
+            verify_cmpr_int(value, wanted);
         }{
-            long value = 562949953421311L;
-            byte[] wanted = { -1, -1, -1, -1, -1, -1, 127 };
-            verify_cmpr_long(value, wanted);
+            byte[] data = { -1, -1, -1, -1, 8 };
+            verify_read_cmpr_int_fails(data);
         }{
-            long value = 562949953421312L;
-            byte[] wanted = { -128, -128, -128, -128, -128, -128, -128, 1 };
-            verify_cmpr_long(value, wanted);
+            byte[] data = { -1, -1, -1, -1, -1, -1, 1 };
+            verify_read_cmpr_int_fails(data);
         }{
-            long value = 72057594037927935L;
-            byte[] wanted = { -1, -1, -1, -1, -1, -1, -1, 127 };
-            verify_cmpr_long(value, wanted);
+            byte[] data = { -1, -1, -1, -1, -1, -1, -1, -1, 1 };
+            verify_read_cmpr_int_fails(data);
         }{
-            long value = 72057594037927936L;
-            byte[] wanted = { -128, -128, -128, -128, -128, -128, -128, -128, 1 };
-            verify_cmpr_long(value, wanted);
-        }{
-            long value = 9223372036854775807L;
-            byte[] wanted = { -1, -1, -1, -1, -1, -1, -1, -1, 127 };
-            verify_cmpr_long(value, wanted);
-        }{
-            long value = -9223372036854775808L;
-            byte[] wanted = { -128, -128, -128, -128, -128, -128, -128, -128, -128, 1 };
-            verify_cmpr_long(value, wanted);
-        }{
-            long value = -1;
-            byte[] wanted = { -1, -1, -1, -1, -1, -1, -1, -1, -1, 1 };
-            verify_cmpr_long(value, wanted);
+            byte[] data = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1 };
+            verify_read_cmpr_int_fails(data);
         }
     }
 
@@ -225,16 +226,16 @@ public class BinaryFormatTestCase {
     @Test
     public void testTypeAndSizeConversion() {
         for (byte type = 0; type < TYPE_LIMIT; ++type) {
-            for (long size = 0; size < 500; ++size) {
+            for (int size = 0; size < 500; ++size) {
                 BufferedOutput expect = new BufferedOutput();
                 BufferedOutput actual = new BufferedOutput();
 
                 if ((size + 1) < META_LIMIT) {
-                    expect.put(encode_type_and_meta((int)type, (int)(size +1)));
+                    expect.put(encode_type_and_meta(type, size +1));
                 } else {
                     expect.put(type);
                     BinaryEncoder encoder = new BinaryEncoder(expect);
-                    encoder.encode_cmpr_long(size);
+                    encoder.encode_cmpr_int(size);
                 }
                 {
                     BinaryEncoder encoder = new BinaryEncoder(actual);
@@ -247,11 +248,13 @@ public class BinaryFormatTestCase {
                 bif.in = new BufferedInput(got);
                 byte b = bif.in.getByte();
                 Type decodedType = decode_type(b);
-                long decodedSize = bif.read_size(decode_meta(b));
+                int decodedSize = bif.in.read_size(decode_meta(b));
                 assertThat(decodedType.ID, is(type));
                 assertThat(decodedSize, is(size));
                 assertThat(bif.in.getConsumedSize(), is(got.length));
                 assertThat(bif.in.failed(), is(false));
+
+                assertThat(BinaryView.extract_children_for_testing(got, 0), is(size));
             }
         }
 
@@ -299,6 +302,12 @@ public class BinaryFormatTestCase {
                         assertThat(decodedBits, is(bits));
                         assertThat(bif.in.getConsumedSize(), is(expect.length));
                         assertThat(bif.in.failed(), is(false));
+
+                        if (hi != 0) {
+                            assertThat(encode_double(BinaryView.extract_double_for_testing(expect, 0)), is(bits));
+                        } else {
+                            assertThat(encode_zigzag(BinaryView.extract_long_for_testing(expect, 0)), is(bits));
+                        }
                     }
                 }
             }
diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java
new file mode 100644
index 00000000000..568124369d4
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java
@@ -0,0 +1,418 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.slime;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.function.Consumer;
+
+import org.junit.Test;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+import static com.yahoo.slime.BinaryFormat.encode_type_and_meta;
+
+public class BinaryViewTest {
+    static String makeString(int size) {
+        var str = new StringBuilder();
+        for (int i = 0; i < size; ++i) {
+            str.append("A");
+        }
+        return str.toString();
+    }
+    static byte[] makeData(int size) {
+        byte[] data = new byte[size];
+        for (int i = 0; i < size; ++i) {
+            data[i] = 65;
+        }
+        return data;
+    }
+    static final int numLeafs = 22;
+    static Cursor insertLeaf(Inserter dst, int id) {
+        return switch (id) {
+        case  0 -> dst.insertNIX();
+        case  1 -> dst.insertBOOL(false);
+        case  2 -> dst.insertBOOL(true);
+        case  3 -> dst.insertLONG(42L);
+        case  4 -> dst.insertLONG(-42L);
+        case  5 -> dst.insertLONG(0x1234_5678_8765_4321L);
+        case  6 -> dst.insertLONG(-0x1234_5678_8765_4321L);
+        case  7 -> dst.insertDOUBLE(3.5);
+        case  8 -> dst.insertDOUBLE(1.0/3.0);
+        case  9 -> dst.insertDOUBLE(-3.5);
+        case 10 -> dst.insertDOUBLE(-(1.0/3.0));
+        case 11 -> dst.insertSTRING(makeString(5));
+        case 12 -> dst.insertSTRING(makeString(50));
+        case 13 -> dst.insertSTRING(makeString(300));
+        case 14 -> dst.insertSTRING(makeData(5));
+        case 15 -> dst.insertSTRING(makeData(50));
+        case 16 -> dst.insertSTRING(makeData(300));
+        case 17 -> dst.insertDATA(makeData(5));
+        case 18 -> dst.insertDATA(makeData(50));
+        case 19 -> dst.insertDATA(makeData(300));
+        case 20 -> dst.insertARRAY();
+        case 21 -> dst.insertOBJECT();
+        default -> NixValue.invalid();
+        };
+    }
+    static Cursor insertInnerObject(Inserter dst) {
+        var obj = dst.insertOBJECT();
+        for (int i = 0; i < numLeafs; ++i) {
+            assertTrue(insertLeaf(new ObjectInserter(obj, "leaf" + i), i).valid());
+        }
+        return obj;
+    }
+    static Cursor insertInnerArray(Inserter dst) {
+        var arr = dst.insertARRAY();
+        for (int i = 0; i < numLeafs; ++i) {
+            assertTrue(insertLeaf(new ArrayInserter(arr), i).valid());
+        }
+        return arr;
+    }
+    static Cursor insertOuterObject(Inserter dst) {
+        var obj = dst.insertOBJECT();
+        assertTrue(insertInnerObject(new ObjectInserter(obj, "foo")).valid());
+        assertTrue(insertInnerArray(new ObjectInserter(obj, "bar")).valid());
+        return obj;
+    }
+    static Cursor insertOuterArray(Inserter dst) {
+        var arr = dst.insertARRAY();
+        assertTrue(insertInnerObject(new ArrayInserter(arr)).valid());
+        assertTrue(insertInnerArray(new ArrayInserter(arr)).valid());
+        return arr;
+    }
+    static Cursor insertManySymbols(Inserter dst) {
+        var obj = dst.insertOBJECT();
+        for (int i = 0; i < 300; ++i) {
+            obj.setLong("val" + i, i);
+        }
+        assertEquals(300, obj.fields());
+        return obj;
+    }
+    static Cursor insertLargeArray(Inserter dst) {
+        var arr = dst.insertARRAY();
+        for (int i = 0; i < 300; ++i) {
+            arr.addLong(i);
+        }
+        assertEquals(300, arr.entries());
+        return arr;
+    }
+    static final int numShapes = numLeafs + 6;
+    static Cursor insertRoot(Slime dst, int shape) {
+        var root = new SlimeInserter(dst);
+        if (shape < numLeafs) {
+            return insertLeaf(root, shape);
+        }
+        return switch (shape) {
+        case (numLeafs) -> insertInnerObject(root);
+        case (numLeafs + 1) -> insertInnerArray(root);
+        case (numLeafs + 2) -> insertOuterObject(root);
+        case (numLeafs + 3) -> insertOuterArray(root); 
+        case (numLeafs + 4) -> insertManySymbols(root);
+        case (numLeafs + 5) -> insertLargeArray(root);
+        default -> NixValue.invalid();
+        };
+    }
+    static Slime makeSlime(int shape) {
+        var slime = new Slime();
+        var root = insertRoot(slime, shape);
+        assertTrue(root.valid());
+        return slime;
+    }
+
+    class MyConsumer implements Consumer<Inspector> {
+        Inspector value = null;
+        @Override public void accept(Inspector value) {
+            assertNull(ctx, this.value);
+            this.value = value;
+        }
+    };
+
+    void checkConsumer(Inspector view) {
+        var consumer = new MyConsumer();
+        view.ifValid(consumer);
+        assertEquals(ctx, view.valid(), consumer.value != null);
+        if (view.valid()) {
+            assertSame(ctx, view, consumer.value);
+        }
+    }
+
+    class MyVisitor implements Visitor {
+        enum Called { NONE, INVALID, NIX, BOOL, LONG, DOUBLE, UTF8, DATA, ARRAY, OBJECT }
+        Called called = Called.NONE;
+        boolean boolValue;
+        long longValue;
+        double doubleValue;
+        byte[] bytes;
+        Inspector stuff;
+        @Override public void visitInvalid() {
+            assertEquals(ctx, Called.NONE, called);
+            called = Called.INVALID;
+        }
+        @Override public void visitNix() {
+            assertEquals(ctx, Called.NONE, called);
+            called = Called.NIX;
+        }
+        @Override public void visitBool(boolean bit) {
+            assertEquals(ctx, Called.NONE, called);
+            called = Called.BOOL;
+            boolValue = bit;
+        }
+        @Override public void visitLong(long l) {
+            assertEquals(ctx, Called.NONE, called);
+            called = Called.LONG;
+            longValue = l;
+        }
+        @Override public void visitDouble(double d) {
+            assertEquals(ctx, Called.NONE, called);
+            called = Called.DOUBLE;
+            doubleValue = d;
+        }
+        @Override public void visitString(String str) {
+            fail(ctx + ", strings are never utf-16 in binary view");
+        }
+        @Override public void visitString(byte[] utf8) {
+            assertEquals(ctx, Called.NONE, called);
+            called = Called.UTF8;
+            bytes = utf8;
+        }
+        @Override public void visitData(byte[] data) {
+            assertEquals(ctx, Called.NONE, called);
+            called = Called.DATA;
+            bytes = data;
+        }
+        @Override public void visitArray(Inspector arr) {
+            assertEquals(ctx, Called.NONE, called);
+            called = Called.ARRAY;
+            stuff = arr;
+        }
+        @Override public void visitObject(Inspector obj) {
+            assertEquals(ctx, Called.NONE, called);
+            called = Called.OBJECT;
+            stuff = obj;
+        }
+    };
+
+    void checkVisitor(Inspector view) {
+        var visitor = new MyVisitor();
+        view.accept(visitor);
+        if (!view.valid()) {
+            assertEquals(ctx, MyVisitor.Called.INVALID, visitor.called);
+            return;
+        }
+        switch (view.type()) {
+            case NIX:
+                assertEquals(ctx, MyVisitor.Called.NIX, visitor.called);
+                break;
+            case BOOL:
+                assertEquals(ctx, MyVisitor.Called.BOOL, visitor.called);
+                assertEquals(ctx, view.asBool(), visitor.boolValue);
+                break;
+            case LONG:
+                assertEquals(ctx, MyVisitor.Called.LONG, visitor.called);
+                assertEquals(ctx, view.asLong(), visitor.longValue);
+                break;
+            case DOUBLE:
+                assertEquals(ctx, MyVisitor.Called.DOUBLE, visitor.called);
+                assertEquals(ctx, view.asDouble(), visitor.doubleValue, 0.0);
+                break;
+            case STRING:
+                assertEquals(ctx, MyVisitor.Called.UTF8, visitor.called);
+                assertArrayEquals(ctx, view.asUtf8(), visitor.bytes);
+                break;
+            case DATA:
+                assertEquals(ctx, MyVisitor.Called.DATA, visitor.called);
+                assertArrayEquals(ctx, view.asData(), visitor.bytes);
+                break;
+            case ARRAY:
+                assertEquals(ctx, MyVisitor.Called.ARRAY, visitor.called);
+                assertSame(ctx, view, visitor.stuff);
+                break;
+            case OBJECT:
+                assertEquals(ctx, MyVisitor.Called.OBJECT, visitor.called);
+                assertSame(ctx, view, visitor.stuff);
+                break;
+            default:
+                fail(ctx + ", should not be reached");
+                break;
+        }
+    }
+
+    class MyArrayTraverser implements ArrayTraverser {
+        ArrayList<Inspector> list = new ArrayList<>();
+        @Override public void entry(int idx, Inspector value) {
+            list.add(value);
+        }
+    }
+
+    void checkTraverseArray(Inspector value, Inspector view) {
+        var a = new MyArrayTraverser();
+        var b = new MyArrayTraverser();
+        value.traverse(a);
+        view.traverse(b);
+        assertEquals(ctx, a.list.size(), b.list.size());
+        for (int i = 0; i < a.list.size(); ++i) {
+            checkParity(a.list.get(i), b.list.get(i));
+        }
+    }
+
+    class MyObjectSymbolTraverser implements ObjectSymbolTraverser {
+        HashMap<Integer,Inspector> map = new HashMap<>();
+        @Override public void field(int sym, Inspector value) {
+            map.put(sym, value);
+        }
+    }
+
+    void checkTraverseObjectSymbol(Inspector value, Inspector view) {
+        var a = new MyObjectSymbolTraverser();
+        var b = new MyObjectSymbolTraverser();
+        value.traverse(a);
+        view.traverse(b);
+        assertEquals(ctx, a.map.size(), b.map.size());
+        for (Integer key: a.map.keySet()) {
+            assertTrue(ctx, b.map.containsKey(key));
+            checkParity(a.map.get(key), b.map.get(key));
+        }
+    }
+
+    class MyObjectTraverser implements ObjectTraverser {
+        HashMap<String,Inspector> map = new HashMap<>();
+        @Override public void field(String name, Inspector value) {
+            map.put(name, value);
+        }
+    }
+
+    void checkTraverseObject(Inspector value, Inspector view) {
+        var a = new MyObjectTraverser();
+        var b = new MyObjectTraverser();
+        value.traverse(a);
+        view.traverse(b);
+        assertEquals(ctx, a.map.size(), b.map.size());
+        for (String key: a.map.keySet()) {
+            assertTrue(ctx, b.map.containsKey(key));
+            checkParity(a.map.get(key), b.map.get(key));
+        }
+    }
+    void checkParity(Inspector value, Inspector view) {
+        checkConsumer(view);
+        checkVisitor(view);
+        if (value == view) {
+            // avoid infinite invalid nix recursion
+            assertSame(ctx, value, view);
+            return;
+        }
+        assertEquals(ctx, value.valid(), view.valid());
+        assertEquals(ctx, value.type(), view.type());
+        assertEquals(ctx, value.children(), view.children());
+        assertEquals(ctx, value.entries(), view.entries());
+        assertEquals(ctx, value.fields(), view.fields());
+        assertEquals(ctx, value.asBool(), view.asBool());
+        assertEquals(ctx, value.asLong(), view.asLong());
+        assertEquals(ctx, value.asDouble(), view.asDouble(), 0.0);
+        assertEquals(ctx, value.asString(), view.asString());
+        assertArrayEquals(ctx, value.asUtf8(), view.asUtf8());
+        assertArrayEquals(ctx, value.asData(), view.asData());
+        checkTraverseArray(value, view);
+        checkTraverseObjectSymbol(value, view);
+        checkTraverseObject(value, view);
+        checkParity(value.entry(0), view.entry(0));
+        checkParity(value.entry(1), view.entry(1));
+        checkParity(value.entry(2), view.entry(2));
+        checkParity(value.entry(3), view.entry(3));
+        checkParity(value.entry(200), view.entry(200));
+        checkParity(value.entry(500), view.entry(500));
+        checkParity(value.entry(-1), view.entry(-1));
+        checkParity(value.field(0), view.field(0));
+        checkParity(value.field(1), view.field(1));
+        checkParity(value.field(2), view.field(2));
+        checkParity(value.field(3), view.field(3));
+        checkParity(value.field(SymbolTable.INVALID), view.field(SymbolTable.INVALID));
+        checkParity(value.field(-1), view.field(-1));
+        checkParity(value.field("foo"), view.field("foo"));
+        checkParity(value.field("bar"), view.field("bar"));
+        checkParity(value.field("val256"), view.field("val256"));
+        checkParity(value.field("bogus"), view.field("bogus"));
+        assertTrue(ctx, value.equalTo(view));
+        assertTrue(ctx, view.equalTo(value));
+    }
+
+    String ctx;
+    @Test public void testBinaryViewShapesParity() {
+        for (int i = 0; i < numShapes; ++i) {
+            var slime = makeSlime(i);
+            ctx = "case " + i + ": '" + slime.toString() + "'";
+            byte[] data = BinaryFormat.encode(slime);
+            try {
+                checkParity(slime.get(), BinaryView.inspect(data));
+            } catch (Exception e) {
+                fail(ctx + ", got exception: " + e);
+            }
+        }
+    }
+
+    @Test public void testTrivialView() {
+        byte[] data = {0, 0};
+        var input = new BufferedInput(data);
+        var view = BinaryView.inspectImpl(input);
+        assertTrue(view.valid());
+        assertEquals(Type.NIX, view.type());
+        assertFalse(input.failed());
+    }
+
+    @Test public void testUnderflow() {
+        byte[] data = {};
+        var input = new BufferedInput(data);
+        var view = BinaryView.inspectImpl(input);
+        assertFalse(view.valid());
+        assertTrue(input.failed());
+        assertEquals("underflow", input.getErrorMessage());
+    }
+
+    @Test public void testMultiByteUnderflow() {
+        byte[] data = { 0, encode_type_and_meta(Type.STRING.ID, 3), 65 };
+        var input = new BufferedInput(data);
+        var view = BinaryView.inspectImpl(input);
+        assertFalse(view.valid());
+        assertTrue(input.failed());
+        assertEquals("underflow", input.getErrorMessage());
+    }
+
+    @Test public void testCompressedIntOverflow() {
+        byte[] data = { -1, -1, -1, -1, 8 };
+        var input = new BufferedInput(data);
+        var view = BinaryView.inspectImpl(input);
+        assertFalse(view.valid());
+        assertTrue(input.failed());
+        assertEquals("compressed int overflow", input.getErrorMessage());
+    }
+
+    @Test public void testExtBitsOverflow() {
+        byte[] data = { 0, encode_type_and_meta(Type.OBJECT.ID, 2), -1, -1, -1, -1, 1 };
+        var input = new BufferedInput(data);
+        var view = BinaryView.inspectImpl(input);
+        assertFalse(view.valid());
+        assertTrue(input.failed());
+        assertEquals("symbol id too big", input.getErrorMessage());
+    }
+
+    @Test public void testDecodeIndexOverflowArray() {
+        byte[] data = { 0, encode_type_and_meta(Type.ARRAY.ID, 4) };
+        var input = new BufferedInput(data);
+        var view = BinaryView.inspectImpl(input);
+        assertFalse(view.valid());
+        assertTrue(input.failed());
+        assertEquals("decode index too big", input.getErrorMessage());
+    }
+
+    @Test public void testDecodeIndexOverflowObject() {
+        byte[] data = { 0, encode_type_and_meta(Type.OBJECT.ID, 4) };
+        var input = new BufferedInput(data);
+        var view = BinaryView.inspectImpl(input);
+        assertFalse(view.valid());
+        assertTrue(input.failed());
+        assertEquals("decode index too big", input.getErrorMessage());
+    }
+}