Add tensor document summary field

14 files changed, 188 insertions, 107 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
index 0e54adae932..7046cb91cd3 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
@@ -16,11 +16,11 @@ import com.yahoo.prelude.hitfield.RawData;
 import com.yahoo.data.access.simple.Value;
 import com.yahoo.data.access.Inspector;
 
-
 /**
- * @author  <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class DataField extends DocsumField implements VariableLengthField {
+
     public DataField(String name) {
         super(name);
     }
@@ -67,4 +67,5 @@ public class DataField extends DocsumField implements VariableLengthField {
     public Object convert(Inspector value) {
         return convert(value.asData(Value.empty().asData()));
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java
index 2f0768d4e8b..3b414e288da 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java
@@ -23,6 +23,7 @@ import java.util.logging.Logger;
  * @author Bjørn Borud
  */
 public final class DocsumDefinitionSet {
+
     public static final int SLIME_MAGIC_ID = 0x55555555;
     private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName());
 
@@ -73,6 +74,7 @@ public final class DocsumDefinitionSet {
         buffer.order(ByteOrder.LITTLE_ENDIAN);
         long docsumClassId = buffer.getInt();
         if (docsumClassId != SLIME_MAGIC_ID) {
+            // TODO: Not used, remove   - bratseth 2017-01-016
             DocsumDefinition docsumDefinition = lookupDocsum(docsumClassId);
             Docsum docsum = new Docsum(docsumDefinition, data);
             hit.addSummary(docsum);
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
index 868420b829d..096382dd023 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
@@ -22,26 +22,26 @@ public abstract class DocsumField {
     private static FieldFactory fieldFactory;
 
     private static class FieldFactory {
+
         Map<String, Constructor<? extends DocsumField>> constructors = new HashMap<>();
 
-        void put(final String typename,
-                final Class<? extends DocsumField> fieldClass)
-                throws NoSuchMethodException, SecurityException
-        {
-            final Constructor<? extends DocsumField> constructor = fieldClass.getConstructor(String.class);
+        void put(String typename, Class<? extends DocsumField> fieldClass) 
+                throws NoSuchMethodException, SecurityException {
+            Constructor<? extends DocsumField> constructor = fieldClass.getConstructor(String.class);
             constructors.put(typename, constructor);
         }
 
-        DocsumField create(final String typename, final String name, final LegacyEmulationConfig emulConfig)
+        DocsumField create(String typename, String name, LegacyEmulationConfig emulConfig)
                 throws InstantiationException, IllegalAccessException,
-                IllegalArgumentException, InvocationTargetException
-        {
+                       IllegalArgumentException, InvocationTargetException {
             DocsumField f = constructors.get(typename).newInstance(name);
             f.emulConfig = emulConfig;
             return f;
         }
     }
+
     private LegacyEmulationConfig emulConfig;
+
     final LegacyEmulationConfig getEmulConfig() { return emulConfig; }
 
     static {
@@ -61,27 +61,27 @@ public abstract class DocsumField {
             fieldFactory.put("jsonstring", StructDataField.class);
             fieldFactory.put("featuredata", FeatureDataField.class);
             fieldFactory.put("xmlstring", XMLField.class);
-        } catch (final Exception e) {
-            log.log(LogLevel.ERROR,
-                    "Could not initialize docsum decoding properly.", e);
+            fieldFactory.put("tensor", TensorField.class);
+        } catch (Exception e) {
+            log.log(LogLevel.ERROR, "Could not initialize docsum decoding properly.", e);
         }
     }
 
     protected String name;
 
-    protected DocsumField(final String name) {
+    protected DocsumField(String name) {
         this.name = name;
     }
 
     /* for unit test only */
-    static DocsumField create(final String name, final String typename) {
+    static DocsumField create(String name, String typename) {
         return create(name, typename, new LegacyEmulationConfig());
     }
 
-    public static DocsumField create(final String name, final String typename, LegacyEmulationConfig emulConfig) {
+    public static DocsumField create(String name, String typename, LegacyEmulationConfig emulConfig) {
         try {
             return fieldFactory.create(typename, name, emulConfig);
-        } catch (final Exception e) {
+        } catch (Exception e) {
             throw new RuntimeException("Unknown field type '" + typename + "'", e);
         }
     }
@@ -90,7 +90,7 @@ public abstract class DocsumField {
         return name;
     }
 
-    public boolean isCompressed(final ByteBuffer b) {
+    public boolean isCompressed(ByteBuffer b) {
         return false;
     }
 
@@ -116,4 +116,5 @@ public abstract class DocsumField {
      * for this field.
      **/
     public abstract Object convert(Inspector value);
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
index 727b4a5f0ba..91e67174c4c 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
@@ -279,10 +279,7 @@ public class FastHit extends Hit {
         for (DocsumField field : docsumDef.getFields()) {
             String fieldName = field.getName();
             if (value.type() == Type.STRING &&
-                (field instanceof LongstringField ||
-                 field instanceof StringField ||
-                 field instanceof XMLField))
-            {
+                (field instanceof LongstringField || field instanceof StringField || field instanceof XMLField)) {
                 setDocsumFieldIfNotPresent(fieldName, new LazyString(field, value));
             } else {
                 Inspector f = value.field(fieldName);
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java
index eb259476aa0..87ee906e592 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java
@@ -103,9 +103,7 @@ public class JSONField extends DocsumField implements VariableLengthField {
         }
     }
 
-    private static class ArrConv extends CompatibilityConverter
-        implements ArrayTraverser
-    {
+    private static class ArrConv extends CompatibilityConverter implements ArrayTraverser {
         @Override
         public void entry(int idx, Inspector value) {
             target.add(stringify(value));
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java
index e9c19590102..d30ab3e87e2 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java
@@ -12,9 +12,8 @@ import com.yahoo.search.result.NanNumber;
 import com.yahoo.data.access.Inspector;
 
 /**
- * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
-
 public class ShortField extends DocsumField {
     static final short EMPTY_VALUE = Short.MIN_VALUE;
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
index 671188e4cae..f5be5ffb678 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
@@ -13,9 +13,10 @@ import com.yahoo.data.access.Inspector;
 
 
 /**
- * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class StringField extends DocsumField implements VariableLengthField {
+
     public StringField(String name) {
         super(name);
     }
@@ -59,4 +60,5 @@ public class StringField extends DocsumField implements VariableLengthField {
     public Object convert(Inspector value) {
         return value.asString("");
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/TensorField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/TensorField.java
new file mode 100644
index 00000000000..422445a31da
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/TensorField.java
@@ -0,0 +1,65 @@
+package com.yahoo.prelude.fastsearch;
+
+import com.yahoo.data.access.Inspector;
+import com.yahoo.data.access.simple.Value;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.serialization.TypedBinaryFormat;
+
+import java.nio.ByteBuffer;
+import java.util.Optional;
+
+/**
+ * A tensor field. Tensors are encoded as a data field where the data (following the length)
+ * is encoded in a tensor binary format defined by com.yahoo.tensor.serialization.TypedBinaryFormat
+ * 
+ * @author bratseth
+ */
+public class TensorField extends DocsumField implements VariableLengthField {
+
+    public TensorField(String name) {
+        super(name);
+    }
+
+    @Override
+    public Tensor decode(ByteBuffer buffer) {
+        int length = ((int) buffer.getShort()) & 0xffff;
+        ByteBuffer contentBuffer = ByteBuffer.wrap(buffer.array(), buffer.arrayOffset() + buffer.position(), length);
+        Tensor tensor = TypedBinaryFormat.decode(Optional.empty(), new GrowableByteBuffer(contentBuffer));
+        buffer.position(buffer.position() + length);
+        return tensor;
+    }
+
+    @Override
+    public Tensor decode(ByteBuffer b, FastHit hit) {
+        Tensor tensor = decode(b);
+        hit.setField(name, tensor);
+        return tensor;
+    }
+
+    @Override
+    public String toString() {
+        return "field " + getName() + " type tensor";
+    }
+
+    @Override
+    public int getLength(ByteBuffer b) {
+        int offset = b.position();
+        int len = ((int) b.getShort()) & 0xffff;
+        b.position(offset + len + (Short.SIZE >> 3));
+        return len + (Short.SIZE >> 3);
+    }
+
+    @Override
+    public int sizeOfLength() {
+        return Short.SIZE >> 3;
+    }
+
+    @Override
+    public Object convert(Inspector value) {
+        byte[] content = value.asData(Value.empty().asData());
+        if (content.length == 0) return null;
+        return TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(content));
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java
index f169533f8db..8b582f555aa 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java
@@ -5,8 +5,10 @@ package com.yahoo.prelude.fastsearch;
  * Interface to easier find the start of the actual data for variable length
  * fields.
  *
- * @author  <a href="mailt:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public interface VariableLengthField {
-    public int sizeOfLength();
+
+    int sizeOfLength();
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java
index 26787e442fc..d9b1a9a57ae 100644
--- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java
+++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java
@@ -2,7 +2,7 @@
 package com.yahoo.prelude.hitfield;
 
 /**
- * A representation of some random data with unknown semantics
+ * A representation of some binary data with unknown semantics
  *
  * @author arnej27959
  */
diff --git a/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java
index 6e750ea0102..34bfefbe98b 100644
--- a/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java
+++ b/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java
@@ -17,9 +17,10 @@ import static junit.framework.TestCase.*;
 /**
  * Tests the PacketDecoder
  *
- * @author  <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ * @author Bjørn Borud
  */
 public class PacketDecoderTestCase {
+
     static byte[] queryResultPacketData
         = new byte[] {0,0,0,104,
                       0,0,0,217-256,
@@ -44,7 +45,6 @@ public class PacketDecoderTestCase {
         data.put(queryResultPacketData);
         data.flip();
 
-
         // not really necessary for testing, but these help visualize
         // the state the buffer should be in so a reader of this test
         // will not have to
diff --git a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java
index 5cf9aeda978..64153363ca5 100644
--- a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java
+++ b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java
@@ -109,7 +109,7 @@ public class RankFeaturesTestCase {
             if (key.contains(".type")) {
                 result.put(key, Utf8.toString(value));
             } else {
-                result.put(key, TypedBinaryFormat.decode(type, value));
+                result.put(key, TypedBinaryFormat.decode(Optional.of(type), GrowableByteBuffer.wrap(value)));
             }
         }
         return result;
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
index 47a3003371e..f45d1e0fd2e 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
@@ -4,81 +4,28 @@ package com.yahoo.prelude.fastsearch;
 
 import com.yahoo.config.subscription.ConfigGetter;
 import com.yahoo.container.search.LegacyEmulationConfig;
-import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
-import com.yahoo.prelude.fastsearch.Docsum;
-import com.yahoo.prelude.fastsearch.DocsumDefinition;
-import com.yahoo.prelude.fastsearch.DocsumDefinitionSet;
-import com.yahoo.prelude.fastsearch.FastHit;
 import com.yahoo.prelude.hitfield.RawData;
 import com.yahoo.prelude.hitfield.XMLString;
 import com.yahoo.prelude.hitfield.JSONString;
 import com.yahoo.search.result.NanNumber;
 import com.yahoo.search.result.StructuredData;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.GlobalId;
 import com.yahoo.slime.*;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 
 import java.nio.charset.StandardCharsets;
 
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.serialization.TypedBinaryFormat;
 import org.junit.Test;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 
 public class SlimeSummaryTestCase {
 
-    public static DocsumDefinitionSet createDocsumDefinitionSet(String configID) {
-        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
-        return new DocsumDefinitionSet(config.documentdb(0));
-    }
-
-    public static DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) {
-        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
-        return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig);
-    }
-
-    public byte[] makeEmptyDocsum() {
-        Slime slime = new Slime();
-        Cursor docsum = slime.setObject();
-        byte[] tmp = BinaryFormat.encode(slime);
-        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
-        buf.order(ByteOrder.LITTLE_ENDIAN);
-        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
-        buf.order(ByteOrder.BIG_ENDIAN);
-        buf.put(tmp);
-        return buf.array();
-    }
-
-    public byte[] makeDocsum() {
-        Slime slime = new Slime();
-        Cursor docsum = slime.setObject();
-        docsum.setLong("integer_field", 4);
-        docsum.setLong("short_field", 2);
-        docsum.setLong("byte_field", 1);
-        docsum.setDouble("float_field", 4.5);
-        docsum.setDouble("double_field", 8.75);
-        docsum.setLong("int64_field", 8);
-        docsum.setString("string_field", "string_value");
-        docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8));
-        docsum.setString("longstring_field", "longstring_value");
-        docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8));
-        docsum.setString("xmlstring_field", "<tag>xmlstring_value</tag>");
-        {
-            Cursor field = docsum.setObject("jsonstring_field");
-            field.setLong("foo", 1);
-            field.setLong("bar", 2);
-        }
-        byte[] tmp = BinaryFormat.encode(slime);
-        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
-        buf.order(ByteOrder.LITTLE_ENDIAN);
-        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
-        buf.order(ByteOrder.BIG_ENDIAN);
-        buf.put(tmp);
-        return buf.array();
-    }
-
     @Test
     public void testDecodingEmpty() {
         String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg";
@@ -87,13 +34,13 @@ public class SlimeSummaryTestCase {
         byte[] docsum = makeEmptyDocsum();
         FastHit hit = new FastHit();
         set.lazyDecode("default", docsum, hit);
-        assertThat(hit.getField("integer_field"), equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("short_field"),   equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("byte_field"),    equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("float_field"),   equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("double_field"),  equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("int64_field"),   equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("string_field"),  equalTo((Object)""));
+        assertThat(hit.getField("integer_field"), equalTo(NanNumber.NaN));
+        assertThat(hit.getField("short_field"),   equalTo(NanNumber.NaN));
+        assertThat(hit.getField("byte_field"),    equalTo(NanNumber.NaN));
+        assertThat(hit.getField("float_field"),   equalTo(NanNumber.NaN));
+        assertThat(hit.getField("double_field"),  equalTo(NanNumber.NaN));
+        assertThat(hit.getField("int64_field"),   equalTo(NanNumber.NaN));
+        assertThat(hit.getField("string_field"),  equalTo(""));
         assertThat(hit.getField("data_field"), instanceOf(RawData.class));
         assertThat(hit.getField("data_field").toString(), equalTo(""));
         assertThat(hit.getField("longstring_field"), equalTo((Object)""));
@@ -103,6 +50,14 @@ public class SlimeSummaryTestCase {
         assertThat(hit.getField("xmlstring_field").toString(), equalTo(""));
         // assertThat(hit.getField("jsonstring_field"), instanceOf(JSONString.class));
         assertThat(hit.getField("jsonstring_field").toString(), equalTo(""));
+        // Empty tensors are represented by null because we don't have type information here to create the right empty tensor
+        assertNull(hit.getField("tensor_field1"));
+        assertNull(hit.getField("tensor_field2"));
+    }
+
+    private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) {
+        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
+        return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig);
     }
 
     @Test
@@ -127,22 +82,39 @@ public class SlimeSummaryTestCase {
         assertThat(hit.getField("xmlstring_field"), equalTo(null));
         assertThat(hit.getField("xmlstring_field"), equalTo(null));
         assertThat(hit.getField("jsonstring_field"), equalTo(null));
+        assertNull(hit.getField("tensor_field1"));
+        assertNull(hit.getField("tensor_field2"));
+    }
+
+    private byte[] makeEmptyDocsum() {
+        Slime slime = new Slime();
+        Cursor docsum = slime.setObject();
+        byte[] tmp = BinaryFormat.encode(slime);
+        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
+        buf.order(ByteOrder.LITTLE_ENDIAN);
+        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
+        buf.order(ByteOrder.BIG_ENDIAN);
+        buf.put(tmp);
+        return buf.array();
     }
 
     @Test
     public void testDecoding() {
+        Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}");
+        Tensor tensor2 = Tensor.from("tensor(x[],y[1]):{{x:0,y:0}:-0.3}");
+                
         String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg";
         DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf);
-        byte[] docsum = makeDocsum();
+        byte[] docsum = makeDocsum(tensor1, tensor2);
         FastHit hit = new FastHit();
         set.lazyDecode("default", docsum, hit);
-        assertThat(hit.getField("integer_field"), equalTo((Object)new Integer(4)));
-        assertThat(hit.getField("short_field"),   equalTo((Object)new Short((short)2)));
-        assertThat(hit.getField("byte_field"),    equalTo((Object)new Byte((byte)1)));
-        assertThat(hit.getField("float_field"),   equalTo((Object)new Float(4.5f)));
-        assertThat(hit.getField("double_field"),  equalTo((Object)new Double(8.75)));
-        assertThat(hit.getField("int64_field"),   equalTo((Object)new Long(8L)));
-        assertThat(hit.getField("string_field"),  equalTo((Object)"string_value"));
+        assertThat(hit.getField("integer_field"), equalTo(4));
+        assertThat(hit.getField("short_field"),   equalTo((short)2));
+        assertThat(hit.getField("byte_field"),    equalTo((byte)1));
+        assertThat(hit.getField("float_field"),   equalTo(4.5f));
+        assertThat(hit.getField("double_field"),  equalTo(8.75));
+        assertThat(hit.getField("int64_field"),   equalTo(8L));
+        assertThat(hit.getField("string_field"),  equalTo("string_value"));
         assertThat(hit.getField("data_field"), instanceOf(RawData.class));
         assertThat(hit.getField("data_field").toString(), equalTo("data_value"));
         assertThat(hit.getField("longstring_field"), equalTo((Object)"longstring_value"));
@@ -168,5 +140,43 @@ public class SlimeSummaryTestCase {
             assertThat(value.field("foo").asLong(), equalTo(1L));
             assertThat(value.field("bar").asLong(), equalTo(2L));
         }
+        assertEquals(tensor1, hit.getField("tensor_field1"));
+        assertEquals(tensor2, hit.getField("tensor_field2"));
+    }
+
+    private DocsumDefinitionSet createDocsumDefinitionSet(String configID) {
+        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
+        return new DocsumDefinitionSet(config.documentdb(0));
+    }
+
+    private byte[] makeDocsum(Tensor tensor1, Tensor tensor2) {
+        Slime slime = new Slime();
+        Cursor docsum = slime.setObject();
+        docsum.setLong("integer_field", 4);
+        docsum.setLong("short_field", 2);
+        docsum.setLong("byte_field", 1);
+        docsum.setDouble("float_field", 4.5);
+        docsum.setDouble("double_field", 8.75);
+        docsum.setLong("int64_field", 8);
+        docsum.setString("string_field", "string_value");
+        docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8));
+        docsum.setString("longstring_field", "longstring_value");
+        docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8));
+        docsum.setString("xmlstring_field", "<tag>xmlstring_value</tag>");
+        {
+            Cursor field = docsum.setObject("jsonstring_field");
+            field.setLong("foo", 1);
+            field.setLong("bar", 2);
+        }
+        docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1));
+        docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2));
+        byte[] tmp = BinaryFormat.encode(slime);
+        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
+        buf.order(ByteOrder.LITTLE_ENDIAN);
+        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
+        buf.order(ByteOrder.BIG_ENDIAN);
+        buf.put(tmp);
+        return buf.array();
     }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg
index a188754db19..e46904b17d0 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg
@@ -3,7 +3,7 @@ documentdb[0].name test
 documentdb[0].summaryclass[1]
 documentdb[0].summaryclass[0].name default
 documentdb[0].summaryclass[0].id 0
-documentdb[0].summaryclass[0].fields[12]
+documentdb[0].summaryclass[0].fields[14]
 documentdb[0].summaryclass[0].fields[0].name integer_field
 documentdb[0].summaryclass[0].fields[0].type integer
 documentdb[0].summaryclass[0].fields[1].name short_field
@@ -28,3 +28,7 @@ documentdb[0].summaryclass[0].fields[10].name xmlstring_field
 documentdb[0].summaryclass[0].fields[10].type xmlstring
 documentdb[0].summaryclass[0].fields[11].name jsonstring_field
 documentdb[0].summaryclass[0].fields[11].type jsonstring
+documentdb[0].summaryclass[0].fields[12].name tensor_field1
+documentdb[0].summaryclass[0].fields[12].type tensor
+documentdb[0].summaryclass[0].fields[13].name tensor_field2
+documentdb[0].summaryclass[0].fields[13].type tensor