Merge pull request #1519 from yahoo/bratseth/tensors-in-summary

Bratseth/tensors in summary

18 files changed, 213 insertions, 133 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
index 0e54adae932..7046cb91cd3 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
@@ -16,11 +16,11 @@ import com.yahoo.prelude.hitfield.RawData;
 import com.yahoo.data.access.simple.Value;
 import com.yahoo.data.access.Inspector;
 
-
 /**
- * @author  <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class DataField extends DocsumField implements VariableLengthField {
+
     public DataField(String name) {
         super(name);
     }
@@ -67,4 +67,5 @@ public class DataField extends DocsumField implements VariableLengthField {
     public Object convert(Inspector value) {
         return convert(value.asData(Value.empty().asData()));
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java
index 2f0768d4e8b..3b414e288da 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java
@@ -23,6 +23,7 @@ import java.util.logging.Logger;
  * @author Bjørn Borud
  */
 public final class DocsumDefinitionSet {
+
     public static final int SLIME_MAGIC_ID = 0x55555555;
     private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName());
 
@@ -73,6 +74,7 @@ public final class DocsumDefinitionSet {
         buffer.order(ByteOrder.LITTLE_ENDIAN);
         long docsumClassId = buffer.getInt();
         if (docsumClassId != SLIME_MAGIC_ID) {
+            // TODO: Not used, remove   - bratseth 2017-01-016
             DocsumDefinition docsumDefinition = lookupDocsum(docsumClassId);
             Docsum docsum = new Docsum(docsumDefinition, data);
             hit.addSummary(docsum);
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
index 3aa02f57a1e..1e14fea26b7 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
@@ -13,8 +13,8 @@ import com.yahoo.container.search.LegacyEmulationConfig;
 import com.yahoo.log.LogLevel;
 
 /**
- * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Bjørn Borud
+ * @author Steinar Knutsen
  */
 public abstract class DocsumField {
 
@@ -22,26 +22,26 @@ public abstract class DocsumField {
     private static FieldFactory fieldFactory;
 
     private static class FieldFactory {
+
         Map<String, Constructor<? extends DocsumField>> constructors = new HashMap<>();
 
-        void put(final String typename,
-                final Class<? extends DocsumField> fieldClass)
-                throws NoSuchMethodException, SecurityException
-        {
-            final Constructor<? extends DocsumField> constructor = fieldClass.getConstructor(String.class);
+        void put(String typename, Class<? extends DocsumField> fieldClass) 
+                throws NoSuchMethodException, SecurityException {
+            Constructor<? extends DocsumField> constructor = fieldClass.getConstructor(String.class);
             constructors.put(typename, constructor);
         }
 
-        DocsumField create(final String typename, final String name, final LegacyEmulationConfig emulConfig)
+        DocsumField create(String typename, String name, LegacyEmulationConfig emulConfig)
                 throws InstantiationException, IllegalAccessException,
-                IllegalArgumentException, InvocationTargetException
-        {
+                       IllegalArgumentException, InvocationTargetException {
             DocsumField f = constructors.get(typename).newInstance(name);
             f.emulConfig = emulConfig;
             return f;
         }
     }
+
     private LegacyEmulationConfig emulConfig;
+
     final LegacyEmulationConfig getEmulConfig() { return emulConfig; }
 
     static {
@@ -61,27 +61,27 @@ public abstract class DocsumField {
             fieldFactory.put("jsonstring", StructDataField.class);
             fieldFactory.put("featuredata", FeatureDataField.class);
             fieldFactory.put("xmlstring", XMLField.class);
-        } catch (final Exception e) {
-            log.log(LogLevel.ERROR,
-                    "Could not initialize docsum decoding properly.", e);
+            fieldFactory.put("tensor", TensorField.class);
+        } catch (Exception e) {
+            log.log(LogLevel.ERROR, "Could not initialize docsum decoding properly.", e);
         }
     }
 
     protected String name;
 
-    protected DocsumField(final String name) {
+    protected DocsumField(String name) {
         this.name = name;
     }
 
     /* for unit test only */
-    static DocsumField create(final String name, final String typename) {
+    static DocsumField create(String name, String typename) {
         return create(name, typename, new LegacyEmulationConfig());
     }
 
-    public static DocsumField create(final String name, final String typename, LegacyEmulationConfig emulConfig) {
+    public static DocsumField create(String name, String typename, LegacyEmulationConfig emulConfig) {
         try {
             return fieldFactory.create(typename, name, emulConfig);
-        } catch (final Exception e) {
+        } catch (Exception e) {
             throw new RuntimeException("Unknown field type '" + typename + "'", e);
         }
     }
@@ -90,7 +90,7 @@ public abstract class DocsumField {
         return name;
     }
 
-    public boolean isCompressed(final ByteBuffer b) {
+    public boolean isCompressed(ByteBuffer b) {
         return false;
     }
 
@@ -106,8 +106,8 @@ public abstract class DocsumField {
     public abstract Object decode(ByteBuffer b);
 
     /**
-     * Get the number of bytes this field occupies in the given buffer and set
-     * the position of the first byte after this field.
+     * Get the number of bytes this field occupies in the given buffer 
+     * AND SET(!) the position to the first byte after this field.
      */
     public abstract int getLength(ByteBuffer b);
 
@@ -116,4 +116,5 @@ public abstract class DocsumField {
      * for this field.
      **/
     public abstract Object convert(Inspector value);
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
index fe0c4a35d1e..91e67174c4c 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
@@ -174,8 +174,7 @@ public class FastHit extends Hit {
     }
 
     /**
-     *
-     * @param useRowInIndexUri Sets whether to use the row in the index uri. See FastSearcher for details.
+     * Sets whether to use the row in the index uri. See FastSearcher for details.
      */
     public void setUseRowInIndexUri(boolean useRowInIndexUri) {
         this.useRowInIndexUri = useRowInIndexUri;
@@ -189,7 +188,7 @@ public class FastHit extends Hit {
     }
 
     /**
-     *  @return the row number where this hit originated, or 0 if not known
+     *  Returns the row number where this hit originated, or 0 if not known
      * */
     public int getRow() {
         if (rowBits == 0) {
@@ -214,17 +213,20 @@ public class FastHit extends Hit {
      * filled returns the following types, even when the field has no actual value:</p>
      *
      * <ul>
-     *     <li><b>Dynamic summary string fields</b>: A Java String before JuniperSearcher and a HitField after.</li>
-     *     <li><b>string/uri/content</b>: A Java String.<br>
+     *     <li><b>string and uri fields</b>: A Java String.<br>
      *     The empty string ("") if no value is assigned in the document.
      *
+     *     <li><b>Dynamic summary string fields</b>: A Java String before JuniperSearcher and a HitField after.</li>
+     *     
      *     <li><b>Numerics</b>: The corresponding numeric Java type.<br>
      *     If the field has <i>no value</i> assigned in the document,
      *     the special numeric {@link com.yahoo.search.result.NanNumber#NaN} is returned.
      *
-     *     <li><b>raw</b>: A {@link com.yahoo.prelude.hitfield.RawData} instance
+     *     <li><b>raw fields</b>: A {@link com.yahoo.prelude.hitfield.RawData} instance
+     *
+     *     <li><b>tensor fields</b>: A {@link com.yahoo.tensor.Tensor} instance
      *
-     *     <li><b>multivalue fields</b>: A {@link com.yahoo.prelude.hitfield.JSONString} instance
+     *     <li><b>multivalue fields</b>: A {@link com.yahoo.data.access.Inspector} instance
      * </ul>
      */
     @Override
@@ -277,10 +279,7 @@ public class FastHit extends Hit {
         for (DocsumField field : docsumDef.getFields()) {
             String fieldName = field.getName();
             if (value.type() == Type.STRING &&
-                (field instanceof LongstringField ||
-                 field instanceof StringField ||
-                 field instanceof XMLField))
-            {
+                (field instanceof LongstringField || field instanceof StringField || field instanceof XMLField)) {
                 setDocsumFieldIfNotPresent(fieldName, new LazyString(field, value));
             } else {
                 Inspector f = value.field(fieldName);
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java
index d61a15723ac..87ee906e592 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java
@@ -1,7 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.fastsearch;
 
-
 import java.nio.ByteBuffer;
 
 import com.yahoo.io.SlowInflate;
@@ -10,13 +9,13 @@ import com.yahoo.text.Utf8;
 import com.yahoo.data.access.*;
 import com.yahoo.data.access.simple.Value;
 
-
 /**
- * Class representing a JSON string field in the result set
+ * A hit field containing JSON structured data
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public class JSONField extends DocsumField implements VariableLengthField {
+
     public JSONField(String name) {
         super(name);
     }
@@ -104,9 +103,7 @@ public class JSONField extends DocsumField implements VariableLengthField {
         }
     }
 
-    private static class ArrConv extends CompatibilityConverter
-        implements ArrayTraverser
-    {
+    private static class ArrConv extends CompatibilityConverter implements ArrayTraverser {
         @Override
         public void entry(int idx, Inspector value) {
             target.add(stringify(value));
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java
index 617f382f462..3ad8ba510aa 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java
@@ -5,7 +5,6 @@
  */
 package com.yahoo.prelude.fastsearch;
 
-
 import java.nio.ByteBuffer;
 
 import com.yahoo.io.SlowInflate;
@@ -13,11 +12,11 @@ import com.yahoo.prelude.hitfield.RawData;
 import com.yahoo.data.access.simple.Value;
 import com.yahoo.data.access.Inspector;
 
-
 /**
- * @author  <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class LongdataField extends DocsumField implements VariableLengthField {
+
     public LongdataField(String name) {
         super(name);
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java
index 744476beaa5..2cb3ac51873 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java
@@ -5,18 +5,17 @@
  */
 package com.yahoo.prelude.fastsearch;
 
-
 import java.nio.ByteBuffer;
 
 import com.yahoo.io.SlowInflate;
 import com.yahoo.text.Utf8;
 import com.yahoo.data.access.Inspector;
 
-
 /**
- * @author  <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class LongstringField extends DocsumField implements VariableLengthField {
+
     public LongstringField(String name) {
         super(name);
     }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java
index e9c19590102..d30ab3e87e2 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java
@@ -12,9 +12,8 @@ import com.yahoo.search.result.NanNumber;
 import com.yahoo.data.access.Inspector;
 
 /**
- * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
-
 public class ShortField extends DocsumField {
     static final short EMPTY_VALUE = Short.MIN_VALUE;
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
index 671188e4cae..f5be5ffb678 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
@@ -13,9 +13,10 @@ import com.yahoo.data.access.Inspector;
 
 
 /**
- * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class StringField extends DocsumField implements VariableLengthField {
+
     public StringField(String name) {
         super(name);
     }
@@ -59,4 +60,5 @@ public class StringField extends DocsumField implements VariableLengthField {
     public Object convert(Inspector value) {
         return value.asString("");
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java
index f0f4b82c22a..c83f209e8f0 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StructDataField.java
@@ -8,7 +8,7 @@ import com.yahoo.container.search.LegacyEmulationConfig;
 import com.yahoo.prelude.hitfield.JSONString;
 
 /**
- * Class representing a XML rendered structured data field in the result set
+ * A hit field containing JSON structured data
  */
 public class StructDataField extends JSONField {
 
@@ -22,9 +22,7 @@ public class StructDataField extends JSONField {
     }
 
     public Object convert(Inspector value) {
-        if (getEmulConfig().stringBackedStructuredData() ||
-            value.type() == Type.STRING)
-        {
+        if (getEmulConfig().stringBackedStructuredData() || value.type() == Type.STRING) {
             return super.convert(value);
         }
         return new StructuredData(value);
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/TensorField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/TensorField.java
new file mode 100644
index 00000000000..0d97a5bdaf7
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/TensorField.java
@@ -0,0 +1,66 @@
+package com.yahoo.prelude.fastsearch;
+
+import com.yahoo.data.access.Inspector;
+import com.yahoo.data.access.simple.Value;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.serialization.TypedBinaryFormat;
+
+import java.nio.ByteBuffer;
+import java.util.Optional;
+
+/**
+ * A tensor field. Tensors are encoded as a data field where the data (following the length)
+ * is encoded in a tensor binary format defined by com.yahoo.tensor.serialization.TypedBinaryFormat
+ * 
+ * @author bratseth
+ */
+public class TensorField extends DocsumField implements VariableLengthField {
+
+    public TensorField(String name) {
+        super(name);
+    }
+
+    @Override
+    public Tensor decode(ByteBuffer buffer) {
+        int length = buffer.getInt();
+        if (length == 0) return null;
+        ByteBuffer contentBuffer = ByteBuffer.wrap(buffer.array(), buffer.arrayOffset() + buffer.position(), length);
+        Tensor tensor = TypedBinaryFormat.decode(Optional.empty(), new GrowableByteBuffer(contentBuffer));
+        buffer.position(buffer.position() + length);
+        return tensor;
+    }
+
+    @Override
+    public Tensor decode(ByteBuffer b, FastHit hit) {
+        Tensor tensor = decode(b);
+        hit.setField(name, tensor);
+        return tensor;
+    }
+
+    @Override
+    public String toString() {
+        return "field " + getName() + " type tensor";
+    }
+
+    @Override
+    public int getLength(ByteBuffer b) {
+        int offset = b.position();
+        int length = b.getInt();
+        b.position(offset + length);
+        return length;
+    }
+
+    @Override
+    public int sizeOfLength() {
+        return 4;
+    }
+
+    @Override
+    public Object convert(Inspector value) {
+        byte[] content = value.asData(Value.empty().asData());
+        if (content.length == 0) return null;
+        return TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(content));
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java
index f169533f8db..2ff7c4864e2 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VariableLengthField.java
@@ -5,8 +5,11 @@ package com.yahoo.prelude.fastsearch;
  * Interface to easier find the start of the actual data for variable length
  * fields.
  *
- * @author  <a href="mailt:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public interface VariableLengthField {
-    public int sizeOfLength();
+
+    /** Returns the size of the length preceeding the data of this field, in bytes */
+    int sizeOfLength();
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java
index 26787e442fc..d9b1a9a57ae 100644
--- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java
+++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawData.java
@@ -2,7 +2,7 @@
 package com.yahoo.prelude.hitfield;
 
 /**
- * A representation of some random data with unknown semantics
+ * A representation of some binary data with unknown semantics
  *
  * @author arnej27959
  */
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java
index d8321a579b3..bd33e3bcd63 100644
--- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java
@@ -134,7 +134,7 @@ import java.util.logging.Logger;
  *     The content of any additional response header fields.
  * </dl>
  *
- * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
+ * @author Arne Bergene Fossaa
  */
 public abstract class HTTPSearcher extends ClusterSearcher<Connection> {
 
diff --git a/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java
index 6e750ea0102..34bfefbe98b 100644
--- a/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java
+++ b/container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java
@@ -17,9 +17,10 @@ import static junit.framework.TestCase.*;
 /**
  * Tests the PacketDecoder
  *
- * @author  <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
+ * @author Bjørn Borud
  */
 public class PacketDecoderTestCase {
+
     static byte[] queryResultPacketData
         = new byte[] {0,0,0,104,
                       0,0,0,217-256,
@@ -44,7 +45,6 @@ public class PacketDecoderTestCase {
         data.put(queryResultPacketData);
         data.flip();
 
-
         // not really necessary for testing, but these help visualize
         // the state the buffer should be in so a reader of this test
         // will not have to
diff --git a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java
index 5cf9aeda978..64153363ca5 100644
--- a/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java
+++ b/container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java
@@ -109,7 +109,7 @@ public class RankFeaturesTestCase {
             if (key.contains(".type")) {
                 result.put(key, Utf8.toString(value));
             } else {
-                result.put(key, TypedBinaryFormat.decode(type, value));
+                result.put(key, TypedBinaryFormat.decode(Optional.of(type), GrowableByteBuffer.wrap(value)));
             }
         }
         return result;
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
index 47a3003371e..f45d1e0fd2e 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java
@@ -4,81 +4,28 @@ package com.yahoo.prelude.fastsearch;
 
 import com.yahoo.config.subscription.ConfigGetter;
 import com.yahoo.container.search.LegacyEmulationConfig;
-import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
-import com.yahoo.prelude.fastsearch.Docsum;
-import com.yahoo.prelude.fastsearch.DocsumDefinition;
-import com.yahoo.prelude.fastsearch.DocsumDefinitionSet;
-import com.yahoo.prelude.fastsearch.FastHit;
 import com.yahoo.prelude.hitfield.RawData;
 import com.yahoo.prelude.hitfield.XMLString;
 import com.yahoo.prelude.hitfield.JSONString;
 import com.yahoo.search.result.NanNumber;
 import com.yahoo.search.result.StructuredData;
-import com.yahoo.document.DocumentId;
-import com.yahoo.document.GlobalId;
 import com.yahoo.slime.*;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 
 import java.nio.charset.StandardCharsets;
 
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.serialization.TypedBinaryFormat;
 import org.junit.Test;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.CoreMatchers.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 
 public class SlimeSummaryTestCase {
 
-    public static DocsumDefinitionSet createDocsumDefinitionSet(String configID) {
-        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
-        return new DocsumDefinitionSet(config.documentdb(0));
-    }
-
-    public static DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) {
-        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
-        return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig);
-    }
-
-    public byte[] makeEmptyDocsum() {
-        Slime slime = new Slime();
-        Cursor docsum = slime.setObject();
-        byte[] tmp = BinaryFormat.encode(slime);
-        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
-        buf.order(ByteOrder.LITTLE_ENDIAN);
-        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
-        buf.order(ByteOrder.BIG_ENDIAN);
-        buf.put(tmp);
-        return buf.array();
-    }
-
-    public byte[] makeDocsum() {
-        Slime slime = new Slime();
-        Cursor docsum = slime.setObject();
-        docsum.setLong("integer_field", 4);
-        docsum.setLong("short_field", 2);
-        docsum.setLong("byte_field", 1);
-        docsum.setDouble("float_field", 4.5);
-        docsum.setDouble("double_field", 8.75);
-        docsum.setLong("int64_field", 8);
-        docsum.setString("string_field", "string_value");
-        docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8));
-        docsum.setString("longstring_field", "longstring_value");
-        docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8));
-        docsum.setString("xmlstring_field", "<tag>xmlstring_value</tag>");
-        {
-            Cursor field = docsum.setObject("jsonstring_field");
-            field.setLong("foo", 1);
-            field.setLong("bar", 2);
-        }
-        byte[] tmp = BinaryFormat.encode(slime);
-        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
-        buf.order(ByteOrder.LITTLE_ENDIAN);
-        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
-        buf.order(ByteOrder.BIG_ENDIAN);
-        buf.put(tmp);
-        return buf.array();
-    }
-
     @Test
     public void testDecodingEmpty() {
         String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg";
@@ -87,13 +34,13 @@ public class SlimeSummaryTestCase {
         byte[] docsum = makeEmptyDocsum();
         FastHit hit = new FastHit();
         set.lazyDecode("default", docsum, hit);
-        assertThat(hit.getField("integer_field"), equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("short_field"),   equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("byte_field"),    equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("float_field"),   equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("double_field"),  equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("int64_field"),   equalTo((Object) NanNumber.NaN));
-        assertThat(hit.getField("string_field"),  equalTo((Object)""));
+        assertThat(hit.getField("integer_field"), equalTo(NanNumber.NaN));
+        assertThat(hit.getField("short_field"),   equalTo(NanNumber.NaN));
+        assertThat(hit.getField("byte_field"),    equalTo(NanNumber.NaN));
+        assertThat(hit.getField("float_field"),   equalTo(NanNumber.NaN));
+        assertThat(hit.getField("double_field"),  equalTo(NanNumber.NaN));
+        assertThat(hit.getField("int64_field"),   equalTo(NanNumber.NaN));
+        assertThat(hit.getField("string_field"),  equalTo(""));
         assertThat(hit.getField("data_field"), instanceOf(RawData.class));
         assertThat(hit.getField("data_field").toString(), equalTo(""));
         assertThat(hit.getField("longstring_field"), equalTo((Object)""));
@@ -103,6 +50,14 @@ public class SlimeSummaryTestCase {
         assertThat(hit.getField("xmlstring_field").toString(), equalTo(""));
         // assertThat(hit.getField("jsonstring_field"), instanceOf(JSONString.class));
         assertThat(hit.getField("jsonstring_field").toString(), equalTo(""));
+        // Empty tensors are represented by null because we don't have type information here to create the right empty tensor
+        assertNull(hit.getField("tensor_field1"));
+        assertNull(hit.getField("tensor_field2"));
+    }
+
+    private DocsumDefinitionSet createDocsumDefinitionSet(String configID, LegacyEmulationConfig legacyEmulationConfig) {
+        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
+        return new DocsumDefinitionSet(config.documentdb(0), legacyEmulationConfig);
     }
 
     @Test
@@ -127,22 +82,39 @@ public class SlimeSummaryTestCase {
         assertThat(hit.getField("xmlstring_field"), equalTo(null));
         assertThat(hit.getField("xmlstring_field"), equalTo(null));
         assertThat(hit.getField("jsonstring_field"), equalTo(null));
+        assertNull(hit.getField("tensor_field1"));
+        assertNull(hit.getField("tensor_field2"));
+    }
+
+    private byte[] makeEmptyDocsum() {
+        Slime slime = new Slime();
+        Cursor docsum = slime.setObject();
+        byte[] tmp = BinaryFormat.encode(slime);
+        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
+        buf.order(ByteOrder.LITTLE_ENDIAN);
+        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
+        buf.order(ByteOrder.BIG_ENDIAN);
+        buf.put(tmp);
+        return buf.array();
     }
 
     @Test
     public void testDecoding() {
+        Tensor tensor1 = Tensor.from("tensor(x{},y{}):{{x:foo,y:bar}:0.1}");
+        Tensor tensor2 = Tensor.from("tensor(x[],y[1]):{{x:0,y:0}:-0.3}");
+                
         String summary_cf = "file:src/test/java/com/yahoo/prelude/fastsearch/summary.cfg";
         DocsumDefinitionSet set = createDocsumDefinitionSet(summary_cf);
-        byte[] docsum = makeDocsum();
+        byte[] docsum = makeDocsum(tensor1, tensor2);
         FastHit hit = new FastHit();
         set.lazyDecode("default", docsum, hit);
-        assertThat(hit.getField("integer_field"), equalTo((Object)new Integer(4)));
-        assertThat(hit.getField("short_field"),   equalTo((Object)new Short((short)2)));
-        assertThat(hit.getField("byte_field"),    equalTo((Object)new Byte((byte)1)));
-        assertThat(hit.getField("float_field"),   equalTo((Object)new Float(4.5f)));
-        assertThat(hit.getField("double_field"),  equalTo((Object)new Double(8.75)));
-        assertThat(hit.getField("int64_field"),   equalTo((Object)new Long(8L)));
-        assertThat(hit.getField("string_field"),  equalTo((Object)"string_value"));
+        assertThat(hit.getField("integer_field"), equalTo(4));
+        assertThat(hit.getField("short_field"),   equalTo((short)2));
+        assertThat(hit.getField("byte_field"),    equalTo((byte)1));
+        assertThat(hit.getField("float_field"),   equalTo(4.5f));
+        assertThat(hit.getField("double_field"),  equalTo(8.75));
+        assertThat(hit.getField("int64_field"),   equalTo(8L));
+        assertThat(hit.getField("string_field"),  equalTo("string_value"));
         assertThat(hit.getField("data_field"), instanceOf(RawData.class));
         assertThat(hit.getField("data_field").toString(), equalTo("data_value"));
         assertThat(hit.getField("longstring_field"), equalTo((Object)"longstring_value"));
@@ -168,5 +140,43 @@ public class SlimeSummaryTestCase {
             assertThat(value.field("foo").asLong(), equalTo(1L));
             assertThat(value.field("bar").asLong(), equalTo(2L));
         }
+        assertEquals(tensor1, hit.getField("tensor_field1"));
+        assertEquals(tensor2, hit.getField("tensor_field2"));
+    }
+
+    private DocsumDefinitionSet createDocsumDefinitionSet(String configID) {
+        DocumentdbInfoConfig config = new ConfigGetter<>(DocumentdbInfoConfig.class).getConfig(configID);
+        return new DocsumDefinitionSet(config.documentdb(0));
+    }
+
+    private byte[] makeDocsum(Tensor tensor1, Tensor tensor2) {
+        Slime slime = new Slime();
+        Cursor docsum = slime.setObject();
+        docsum.setLong("integer_field", 4);
+        docsum.setLong("short_field", 2);
+        docsum.setLong("byte_field", 1);
+        docsum.setDouble("float_field", 4.5);
+        docsum.setDouble("double_field", 8.75);
+        docsum.setLong("int64_field", 8);
+        docsum.setString("string_field", "string_value");
+        docsum.setData("data_field", "data_value".getBytes(StandardCharsets.UTF_8));
+        docsum.setString("longstring_field", "longstring_value");
+        docsum.setData("longdata_field", "longdata_value".getBytes(StandardCharsets.UTF_8));
+        docsum.setString("xmlstring_field", "<tag>xmlstring_value</tag>");
+        {
+            Cursor field = docsum.setObject("jsonstring_field");
+            field.setLong("foo", 1);
+            field.setLong("bar", 2);
+        }
+        docsum.setData("tensor_field1", TypedBinaryFormat.encode(tensor1));
+        docsum.setData("tensor_field2", TypedBinaryFormat.encode(tensor2));
+        byte[] tmp = BinaryFormat.encode(slime);
+        ByteBuffer buf = ByteBuffer.allocate(tmp.length + 4);
+        buf.order(ByteOrder.LITTLE_ENDIAN);
+        buf.putInt(DocsumDefinitionSet.SLIME_MAGIC_ID);
+        buf.order(ByteOrder.BIG_ENDIAN);
+        buf.put(tmp);
+        return buf.array();
     }
+
 }
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg
index a188754db19..e46904b17d0 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/summary.cfg
@@ -3,7 +3,7 @@ documentdb[0].name test
 documentdb[0].summaryclass[1]
 documentdb[0].summaryclass[0].name default
 documentdb[0].summaryclass[0].id 0
-documentdb[0].summaryclass[0].fields[12]
+documentdb[0].summaryclass[0].fields[14]
 documentdb[0].summaryclass[0].fields[0].name integer_field
 documentdb[0].summaryclass[0].fields[0].type integer
 documentdb[0].summaryclass[0].fields[1].name short_field
@@ -28,3 +28,7 @@ documentdb[0].summaryclass[0].fields[10].name xmlstring_field
 documentdb[0].summaryclass[0].fields[10].type xmlstring
 documentdb[0].summaryclass[0].fields[11].name jsonstring_field
 documentdb[0].summaryclass[0].fields[11].type jsonstring
+documentdb[0].summaryclass[0].fields[12].name tensor_field1
+documentdb[0].summaryclass[0].fields[12].type tensor
+documentdb[0].summaryclass[0].fields[13].name tensor_field2
+documentdb[0].summaryclass[0].fields[13].type tensor