Merge with master

11 files changed, 1013 insertions, 15 deletions

diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java
index 4c15b6e2365..6c5dd5e3ba5 100644
--- a/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java
+++ b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java
@@ -45,4 +45,5 @@ public class DaemonThreadFactory implements ThreadFactory {
         }
         return t;
     }
+
 }
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
index 6750c99bf98..c207dabca3a 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java
@@ -125,8 +125,12 @@ public class IndexedTensor implements Tensor {
         if (indexes.length == 0) return 0; // for speed
 
         int valueIndex = 0;
-        for (int i = 0; i < indexes.length; i++)
+        for (int i = 0; i < indexes.length; i++) {
+            if (indexes[i] >= sizes.size(i)) {
+                throw new IndexOutOfBoundsException();
+            }
             valueIndex += productOfDimensionsAfter(i, sizes) * indexes[i];
+        }
         return valueIndex;
     }
 
@@ -134,8 +138,12 @@ public class IndexedTensor implements Tensor {
         if (address.isEmpty()) return 0;
 
         int valueIndex = 0;
-        for (int i = 0; i < address.size(); i++)
+        for (int i = 0; i < address.size(); i++) {
+            if (address.intLabel(i) >= sizes.size(i)) {
+                throw new IndexOutOfBoundsException();
+            }
             valueIndex += productOfDimensionsAfter(i, sizes) * address.intLabel(i);
+        }
         return valueIndex;
     }
 
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java
new file mode 100644
index 00000000000..79bb27fcd1b
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java
@@ -0,0 +1,441 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.tensor;
+
+import com.google.common.annotations.Beta;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * A mixed tensor type. This is class is currently suitable for serialization
+ * and deserialization, not yet for computation.
+ *
+ * A mixed tensor has a combination of mapped and indexed dimensions. By
+ * reordering the mapped dimensions before the indexed dimensions, one can
+ * think of mixed tensors as the mapped dimensions mapping to a
+ * dense tensor. This dense tensor is called a dense subspace.
+ *
+ * @author lesters
+ */
+@Beta
+public class MixedTensor implements Tensor {
+
+    /** The dimension specification for this tensor */
+    private final TensorType type;
+
+    /** The list of cells in the tensor */
+    private final ImmutableList<Cell> cells;
+
+    /** An index structure over the cell list */
+    private final Index index;
+
+    private MixedTensor(TensorType type, ImmutableList<Cell> cells, Index index) {
+        this.type = type;
+        this.cells = ImmutableList.copyOf(cells);
+        this.index = index;
+    }
+
+    /** Returns the tensor type */
+    @Override
+    public TensorType type() { return type; }
+
+    /** Returns the size of the tensor measured in number of cells */
+    @Override
+    public int size() { return cells.size(); }
+
+    /** Returns the value at the given address */
+    @Override
+    public double get(TensorAddress address) {
+        int cellIndex = index.indexOf(address);
+        Cell cell = cells.get(cellIndex);
+        if (!address.equals(cell.getKey())) {
+            throw new IllegalStateException("Unable to find correct cell by direct index.");
+        }
+        return cell.getValue();
+    }
+
+    /**
+     * Returns an iterator over the cells of this tensor.
+     * Cells are returned in order of increasing indexes in the
+     * indexed dimensions, increasing indexes of later dimensions
+     * in the dimension type before earlier. No guarantee is
+     * given for the order of sparse dimensions.
+     */
+    @Override
+    public Iterator<Cell> cellIterator() {
+        return cells.iterator();
+    }
+
+    /**
+     * Returns an iterator over the values of this tensor.
+     * The iteration order is the same as for cellIterator.
+     */
+    @Override
+    public Iterator<Double> valueIterator() {
+        return new Iterator<Double>() {
+            Iterator<Cell> cellIterator = cellIterator();
+            @Override
+            public boolean hasNext() {
+                return cellIterator.hasNext();
+            }
+            @Override
+            public Double next() {
+                return cellIterator.next().getValue();
+            }
+        };
+    }
+
+    @Override
+    public Map<TensorAddress, Double> cells() {
+        ImmutableMap.Builder<TensorAddress, Double> builder = new ImmutableMap.Builder<>();
+        for (Cell cell : cells) {
+            builder.put(cell.getKey(), cell.getValue());
+        }
+        return builder.build();
+    }
+
+    @Override
+    public int hashCode() { return cells.hashCode(); }
+
+    @Override
+    public String toString() { return Tensor.toStandardString(this); }
+
+    @Override
+    public boolean equals(Object other) {
+        if ( ! ( other instanceof Tensor)) return false;
+        return Tensor.equals(this, ((Tensor)other));
+    }
+
+    /** Returns the size of dense subspaces */
+    public int denseSubspaceSize() {
+        return index.denseSubspaceSize();
+    }
+
+    
+    /**
+     * Base class for building mixed tensors.
+     */
+    public abstract static class Builder implements Tensor.Builder {
+
+        final TensorType type;
+
+        /**
+         * Create a builder depending upon the type of indexed dimensions.
+         * If at least one indexed dimension is unbound, we create
+         * a temporary structure while finding dimension bounds.
+         */
+        public static Builder of(TensorType type) {
+            if (type.dimensions().stream().anyMatch(d -> d instanceof TensorType.IndexedUnboundDimension)) {
+                return new UnboundBuilder(type);
+            } else {
+                return new BoundBuilder(type);
+            }
+        }
+
+        private Builder(TensorType type) {
+            this.type = type;
+        }
+
+        @Override
+        public TensorType type() {
+            return type;
+        }
+
+        @Override
+        public Tensor.Builder cell(double value, int... labels) {
+            throw new UnsupportedOperationException("Not implemented.");
+        }
+
+        @Override
+        public CellBuilder cell() {
+            return new CellBuilder(type(), this);
+        }
+
+        @Override
+        public abstract MixedTensor build();
+
+    }
+
+
+    /**
+     * Builder for mixed tensors with bound indexed dimensions.
+     */
+    public static class BoundBuilder extends Builder {
+
+        /** For each sparse partial address, hold a dense subspace */
+        final private Map<TensorAddress, double[]> denseSubspaceMap = new HashMap<>();
+        final private Index.Builder indexBuilder;
+        final private Index index;
+
+        private BoundBuilder(TensorType type) {
+            super(type);
+            indexBuilder = new Index.Builder(type);
+            index = indexBuilder.index();
+        }
+
+        public int denseSubspaceSize() {
+            return index.denseSubspaceSize();
+        }
+
+        private double[] denseSubspace(TensorAddress sparsePartial) {
+            if (!denseSubspaceMap.containsKey(sparsePartial)) {
+                denseSubspaceMap.put(sparsePartial, new double[denseSubspaceSize()]);
+            }
+            return denseSubspaceMap.get(sparsePartial);
+        }
+
+        @Override
+        public Tensor.Builder cell(TensorAddress address, double value) {
+            TensorAddress sparsePart = index.sparsePartialAddress(address);
+            int denseOffset = index.denseOffset(address);
+            double[] denseSubspace = denseSubspace(sparsePart);
+            denseSubspace[denseOffset] = value;
+            return this;
+        }
+
+        public Tensor.Builder block(TensorAddress sparsePart, double[] values) {
+            double[] denseSubspace = denseSubspace(sparsePart);
+            System.arraycopy(values, 0, denseSubspace, 0, denseSubspaceSize());
+            return this;
+        }
+
+        @Override
+        public MixedTensor build() {
+            int count = 0;
+            ImmutableList.Builder<Cell> builder = new ImmutableList.Builder<>();
+
+            for (Map.Entry<TensorAddress, double[]> entry : denseSubspaceMap.entrySet()) {
+                TensorAddress sparsePart = entry.getKey();
+                indexBuilder.put(sparsePart, count);
+
+                double[] denseSubspace = entry.getValue();
+                for (int offset = 0; offset < denseSubspace.length; ++offset) {
+                    TensorAddress cellAddress = index.addressOf(sparsePart, offset);
+                    double value = denseSubspace[offset];
+                    builder.add(new Cell(cellAddress, value));
+                    count++;
+                }
+            }
+            return new MixedTensor(type, builder.build(), indexBuilder.build());
+        }
+
+    }
+
+
+    /**
+     * Temporarily stores all cells to find bounds of indexed dimensions,
+     * then creates a tensor using BoundBuilder. This is due to the
+     * fact that for serialization the size of the dense subspace must be
+     * known, and equal for all dense subspaces. A side effect is that the
+     * tensor type is effectively changed, such that unbound indexed
+     * dimensions become bound.
+     */
+    public static class UnboundBuilder extends Builder {
+
+        private Map<TensorAddress, Double> cells;
+        private final int[] dimensionBounds;
+
+        private UnboundBuilder(TensorType type) {
+            super(type);
+            cells = new HashMap<>();
+            dimensionBounds = new int[type.dimensions().size()];
+        }
+
+        @Override
+        public Tensor.Builder cell(TensorAddress address, double value) {
+            cells.put(address, value);
+            trackBounds(address);
+            return this;
+        }
+
+        @Override
+        public MixedTensor build() {
+            TensorType boundType = createBoundType();
+            BoundBuilder builder = new BoundBuilder(boundType);
+            for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) {
+                builder.cell(cell.getKey(), cell.getValue());
+            }
+            return builder.build();
+        }
+
+        public void trackBounds(TensorAddress address) {
+            for (int i = 0; i < type.dimensions().size(); ++i) {
+                TensorType.Dimension dimension = type.dimensions().get(i);
+                if (dimension.isIndexed()) {
+                    dimensionBounds[i] = Math.max(address.intLabel(i), dimensionBounds[i]);
+                }
+            }
+        }
+
+        public TensorType createBoundType() {
+            TensorType.Builder typeBuilder = new TensorType.Builder();
+            for (int i = 0; i < type.dimensions().size(); ++i) {
+                TensorType.Dimension dimension = type.dimensions().get(i);
+                if (!dimension.isIndexed()) {
+                    typeBuilder.mapped(dimension.name());
+                } else {
+                    int size = dimension.size().orElse(dimensionBounds[i] + 1);
+                    typeBuilder.indexed(dimension.name(), size);
+                }
+            }
+            return typeBuilder.build();
+        }
+        
+    }
+
+    /**
+     * An immutable index into a list of cells.
+     * Contains additional information required
+     * for handling mixed tensor addresses.
+     * Assumes indexed dimensions are bound.
+     */
+    private static class Index {
+
+        private final TensorType type;
+        private final TensorType sparseType;
+        private final TensorType denseType;
+        private final List<TensorType.Dimension> mappedDimensions;
+        private final List<TensorType.Dimension> indexedDimensions;
+
+        private ImmutableMap<TensorAddress, Integer> sparseMap;
+        private int denseSubspaceSize = -1;
+
+        private Index(TensorType type) {
+            this.type = type;
+            this.mappedDimensions = type.dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList());
+            this.indexedDimensions = type.dimensions().stream().filter(d -> d.isIndexed()).collect(Collectors.toList());
+            this.sparseType = createPartialType(mappedDimensions);
+            this.denseType = createPartialType(indexedDimensions);
+        }
+
+        public int indexOf(TensorAddress address) {
+            TensorAddress sparsePart = sparsePartialAddress(address);
+            if (!sparseMap.containsKey(sparsePart)) {
+                throw new IllegalArgumentException("Address not found");
+            }
+            int base = sparseMap.get(sparsePart);
+            int offset = denseOffset(address);
+            return base + offset;
+        }
+
+        public static class Builder {
+            private final Index index;
+            private final ImmutableMap.Builder<TensorAddress, Integer> builder;
+
+            public Builder(TensorType type) {
+                index = new Index(type);
+                builder = new ImmutableMap.Builder<>();
+            }
+
+            public void put(TensorAddress address, int index) {
+                builder.put(address, index);
+            }
+
+            public Index build() {
+                index.sparseMap = builder.build();
+                return index;
+            }
+
+            public Index index() {
+                return index;
+            }
+        }
+
+        public int denseSubspaceSize() {
+            if (denseSubspaceSize == -1) {
+                denseSubspaceSize = 1;
+                for (int i = 0; i < type.dimensions().size(); ++i) {
+                    TensorType.Dimension dimension = type.dimensions().get(i);
+                    if (dimension.isIndexed()) {
+                        denseSubspaceSize *= dimension.size().orElseThrow(() ->
+                                new IllegalArgumentException("Unknown size of indexed dimension."));
+                    }
+                }
+            }
+            return denseSubspaceSize;
+        }
+        
+        private TensorAddress sparsePartialAddress(TensorAddress address) {
+            if (type.dimensions().size() != address.size()) {
+                throw new IllegalArgumentException("Tensor type and address are not of same size.");
+            }
+            TensorAddress.Builder builder = new TensorAddress.Builder(sparseType);
+            for (int i = 0; i < type.dimensions().size(); ++i) {
+                TensorType.Dimension dimension = type.dimensions().get(i);
+                if (!dimension.isIndexed()) {
+                    builder.add(dimension.name(), address.label(i));
+                }
+            }
+            return builder.build();
+        }
+
+        private int denseOffset(TensorAddress address) {
+            int innerSize = 1;
+            int offset = 0;
+            for (int i = type.dimensions().size(); --i >= 0; ) {
+                TensorType.Dimension dimension = type.dimensions().get(i);
+                if (dimension.isIndexed()) {
+                    int label = address.intLabel(i);
+                    offset += label * innerSize;
+                    innerSize *= dimension.size().orElseThrow(() ->
+                            new IllegalArgumentException("Unknown size of indexed dimension."));
+                }
+            }
+            return offset;
+        }
+
+        private TensorAddress denseOffsetToAddress(int denseOffset) {
+            if (denseOffset < 0 || denseOffset > denseSubspaceSize) {
+                throw new IllegalArgumentException("Offset out of bounds");
+            }
+
+            int restSize = denseOffset;
+            int innerSize = denseSubspaceSize;
+            int[] labels = new int[indexedDimensions.size()];
+
+            for (int i = 0; i < labels.length; ++i) {
+                TensorType.Dimension dimension = indexedDimensions.get(i);
+                int dimensionSize = dimension.size().orElseThrow(() ->
+                        new IllegalArgumentException("Unknown size of indexed dimension."));
+
+                innerSize /= dimensionSize;
+                labels[i] = restSize / innerSize;
+                restSize %= innerSize;
+            }
+            return TensorAddress.of(labels);
+        }
+
+        private TensorAddress addressOf(TensorAddress sparsePart, int denseOffset) {
+            TensorAddress densePart = denseOffsetToAddress(denseOffset);
+            String[] labels = new String[type.dimensions().size()];
+            int mappedIndex = 0;
+            int indexedIndex = 0;
+            for (TensorType.Dimension d : type.dimensions()) {
+                if (d.isIndexed()) {
+                    labels[mappedIndex + indexedIndex] = densePart.label(indexedIndex);
+                    indexedIndex++;
+                } else {
+                    labels[mappedIndex + indexedIndex] = sparsePart.label(mappedIndex);
+                    mappedIndex++;
+                }
+            }
+            return TensorAddress.of(labels);
+        }
+
+    }
+
+    public static TensorType createPartialType(List<TensorType.Dimension> dimensions) {
+        TensorType.Builder builder = new TensorType.Builder();
+        for (TensorType.Dimension dimension : dimensions) {
+            builder.set(dimension);
+        }
+        return builder.build();
+    }
+
+}
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java
index 5e3af70cba4..2ed211539d8 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java
@@ -171,12 +171,16 @@ public interface Tensor {
     default Tensor max(Tensor argument) { return join(argument, (a, b) -> (a > b ? a : b )); }
     default Tensor min(Tensor argument) { return join(argument, (a, b) -> (a < b ? a : b )); }
     default Tensor atan2(Tensor argument) { return join(argument, Math::atan2); }
+    default Tensor pow(Tensor argument) { return join(argument, Math::pow); }
+    default Tensor fmod(Tensor argument) { return join(argument, (a, b) -> ( a % b )); }
+    default Tensor ldexp(Tensor argument) { return join(argument, (a, b) -> ( a * Math.pow(2.0, (int)b) )); }
     default Tensor larger(Tensor argument) { return join(argument, (a, b) -> ( a > b ? 1.0 : 0.0)); }
     default Tensor largerOrEqual(Tensor argument) { return join(argument, (a, b) -> ( a >= b ? 1.0 : 0.0)); }
     default Tensor smaller(Tensor argument) { return join(argument, (a, b) -> ( a < b ? 1.0 : 0.0)); }
     default Tensor smallerOrEqual(Tensor argument) { return join(argument, (a, b) -> ( a <= b ? 1.0 : 0.0)); }
     default Tensor equal(Tensor argument) { return join(argument, (a, b) -> ( a == b ? 1.0 : 0.0)); }
     default Tensor notEqual(Tensor argument) { return join(argument, (a, b) -> ( a != b ? 1.0 : 0.0)); }
+    default Tensor approxEqual(Tensor argument) { return join(argument, (a, b) -> ( approxEquals(a,b) ? 1.0 : 0.0)); }
 
     default Tensor avg(String dimension) { return avg(Collections.singletonList(dimension)); }
     default Tensor avg(List<String> dimensions) { return reduce(Reduce.Aggregator.avg, dimensions); }
@@ -259,11 +263,27 @@ public interface Tensor {
         if ( a.size() != b.size()) return false;
         for (Iterator<Cell> aIterator = a.cellIterator(); aIterator.hasNext(); ) {
             Cell aCell = aIterator.next();
-            if ( ! aCell.getValue().equals(b.get(aCell.getKey()))) return false;
+            double aValue = aCell.getValue();
+            double bValue = b.get(aCell.getKey());
+            if (!approxEquals(aValue, bValue, 1e-6)) return false;
         }
         return true;
     }
 
+    static boolean approxEquals(double x, double y, double tolerance) {
+        return Math.abs(x-y) < tolerance;
+    }
+
+    static boolean approxEquals(double x, double y) {
+        if (y < -1.0 || y > 1.0) {
+            x = Math.nextAfter(x/y, 1.0);
+            y = 1.0;
+        } else {
+            x = Math.nextAfter(x, y);
+        }
+        return x==y;
+    }
+
     // ----------------- Factories
 
     /**
@@ -347,7 +367,7 @@ public interface Tensor {
             boolean containsIndexed = type.dimensions().stream().anyMatch(d -> d.isIndexed());
             boolean containsMapped = type.dimensions().stream().anyMatch( d ->  ! d.isIndexed());
             if (containsIndexed && containsMapped)
-                throw new IllegalArgumentException("Combining indexed and mapped dimensions is not supported yet");
+                return MixedTensor.Builder.of(type);
             if (containsMapped)
                 return MappedTensor.Builder.of(type);
             else // indexed or empty
@@ -359,7 +379,7 @@ public interface Tensor {
             boolean containsIndexed = type.dimensions().stream().anyMatch(d -> d.isIndexed());
             boolean containsMapped = type.dimensions().stream().anyMatch( d ->  ! d.isIndexed());
             if (containsIndexed && containsMapped)
-                throw new IllegalArgumentException("Combining indexed and mapped dimensions is not supported yet");
+                return MixedTensor.Builder.of(type);
             if (containsMapped)
                 return MappedTensor.Builder.of(type);
             else // indexed or empty
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java
index 401f9a10eda..1dbb94fdb20 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java
@@ -134,9 +134,7 @@ public class Concat extends PrimitiveTensorFunction {
             if (currentDimension.equals(concatDimension))
                 concatSizes.set(i, aSize + bSize);
             else if (aSize != 0 && bSize != 0 && aSize!=bSize )
-                throw new IllegalArgumentException("Dimension " + currentDimension + " must be of the same size when " +
-                                                   "concatenating " + a.type() + " and " + b.type() + " along dimension " + 
-                                                   concatDimension + ", but was " + aSize + " and " + bSize);
+                concatSizes.set(i, Math.min(aSize, bSize));
             else
                 concatSizes.set(i, Math.max(aSize, bSize));
         }
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java
new file mode 100644
index 00000000000..61dfa888567
--- /dev/null
+++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java
@@ -0,0 +1,129 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.tensor.serialization;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.tensor.MixedTensor;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorAddress;
+import com.yahoo.tensor.TensorType;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+/**
+ * Implementation of a mixed binary format for a tensor.
+ * See eval/src/vespa/eval/tensor/serialization/format.txt for format.
+ *
+ * @author lesters
+ */
+@Beta
+class MixedBinaryFormat implements BinaryFormat {
+
+    @Override
+    public void encode(GrowableByteBuffer buffer, Tensor tensor) {
+        if ( ! ( tensor instanceof MixedTensor))
+            throw new RuntimeException("The mixed format is only supported for mixed tensors");
+        MixedTensor mixed = (MixedTensor) tensor;
+        encodeSparseDimensions(buffer, mixed);
+        encodeDenseDimensions(buffer, mixed);
+        encodeCells(buffer, mixed);
+    }
+
+    private void encodeSparseDimensions(GrowableByteBuffer buffer, MixedTensor tensor) {
+        List<TensorType.Dimension> sparseDimensions = tensor.type().dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList());
+        buffer.putInt1_4Bytes(sparseDimensions.size());
+        for (TensorType.Dimension dimension : sparseDimensions) {
+            buffer.putUtf8String(dimension.name());
+        }
+    }
+
+    private void encodeDenseDimensions(GrowableByteBuffer buffer, MixedTensor tensor) {
+        List<TensorType.Dimension> denseDimensions = tensor.type().dimensions().stream().filter(d -> d.isIndexed()).collect(Collectors.toList());
+        buffer.putInt1_4Bytes(denseDimensions.size());
+        for (TensorType.Dimension dimension : denseDimensions) {
+            buffer.putUtf8String(dimension.name());
+            buffer.putInt1_4Bytes(dimension.size().orElseThrow(() ->
+                    new IllegalArgumentException("Unknown size of indexed dimension.")));
+        }
+    }
+
+    private void encodeCells(GrowableByteBuffer buffer, MixedTensor tensor) {
+        List<TensorType.Dimension> sparseDimensions = tensor.type().dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList());
+        int denseSubspaceSize = tensor.denseSubspaceSize();
+        if (sparseDimensions.size() > 0) {
+            buffer.putInt1_4Bytes(tensor.size() / denseSubspaceSize);
+        }
+        Iterator<Tensor.Cell> cellIterator = tensor.cellIterator();
+        while (cellIterator.hasNext()) {
+            Tensor.Cell cell = cellIterator.next();
+            for (TensorType.Dimension dimension : sparseDimensions) {
+                int index = tensor.type().indexOfDimension(dimension.name()).orElseThrow(() ->
+                    new IllegalStateException("Dimension not found in address."));
+                buffer.putUtf8String(cell.getKey().label(index));
+            }
+            buffer.putDouble(cell.getValue());
+            for (int i = 1; i < denseSubspaceSize; ++i ) {
+                buffer.putDouble(cellIterator.next().getValue());
+            }
+        }
+    }
+
+    @Override
+    public Tensor decode(Optional<TensorType> optionalType, GrowableByteBuffer buffer) {
+        TensorType type;
+        if (optionalType.isPresent()) {
+            type = optionalType.get();
+            TensorType serializedType = decodeType(buffer);
+            if ( ! serializedType.isAssignableTo(type))
+                throw new IllegalArgumentException("Type/instance mismatch: A tensor of type " + serializedType +
+                        " cannot be assigned to type " + type);
+        }
+        else {
+            type = decodeType(buffer);
+        }
+        MixedTensor.BoundBuilder builder = (MixedTensor.BoundBuilder)MixedTensor.Builder.of(type);
+        decodeCells(buffer, builder, type);
+        return builder.build();
+    }
+
+    private TensorType decodeType(GrowableByteBuffer buffer) {
+        TensorType.Builder builder = new TensorType.Builder();
+        int numMappedDimensions = buffer.getInt1_4Bytes();
+        for (int i = 0; i < numMappedDimensions; ++i) {
+            builder.mapped(buffer.getUtf8String());
+        }
+        int numIndexedDimensions = buffer.getInt1_4Bytes();
+        for (int i = 0; i < numIndexedDimensions; ++i) {
+            builder.indexed(buffer.getUtf8String(), buffer.getInt1_4Bytes());
+        }
+        return builder.build();
+    }
+
+    private void decodeCells(GrowableByteBuffer buffer, MixedTensor.BoundBuilder builder, TensorType type) {
+        List<TensorType.Dimension> sparseDimensions = type.dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList());
+        TensorType sparseType = MixedTensor.createPartialType(sparseDimensions);
+        int denseSubspaceSize = builder.denseSubspaceSize();
+
+        int numBlocks = 1;
+        if (sparseDimensions.size() > 0) {
+            numBlocks = buffer.getInt1_4Bytes();
+        }
+
+        double[] denseSubspace = new double[denseSubspaceSize];
+        for (int i = 0; i < numBlocks; ++i) {
+            TensorAddress.Builder sparseAddress = new TensorAddress.Builder(sparseType);
+            for (TensorType.Dimension sparseDimension : sparseDimensions) {
+                sparseAddress.add(sparseDimension.name(), buffer.getUtf8String());
+            }
+            for (int denseOffset = 0; denseOffset < denseSubspaceSize; denseOffset++) {
+                denseSubspace[denseOffset] = buffer.getDouble();
+            }
+            builder.block(sparseAddress.build(), denseSubspace);
+        }
+    }
+
+}
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java
index 657d262b401..7467554790a 100644
--- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java
+++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java
@@ -4,6 +4,7 @@ package com.yahoo.tensor.serialization;
 import com.google.common.annotations.Beta;
 import com.yahoo.io.GrowableByteBuffer;
 import com.yahoo.tensor.IndexedTensor;
+import com.yahoo.tensor.MixedTensor;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.TensorType;
 
@@ -23,10 +24,15 @@ public class TypedBinaryFormat {
 
     private static final int SPARSE_BINARY_FORMAT_TYPE = 1;
     private static final int DENSE_BINARY_FORMAT_TYPE = 2;
+    private static final int MIXED_BINARY_FORMAT_TYPE = 3;
 
     public static byte[] encode(Tensor tensor) {
         GrowableByteBuffer buffer = new GrowableByteBuffer();
-        if (tensor instanceof IndexedTensor) {
+        if (tensor instanceof MixedTensor) {
+            buffer.putInt1_4Bytes(MIXED_BINARY_FORMAT_TYPE);
+            new MixedBinaryFormat().encode(buffer, tensor);
+        }
+        else if (tensor instanceof IndexedTensor) {
             buffer.putInt1_4Bytes(DENSE_BINARY_FORMAT_TYPE);
             new DenseBinaryFormat().encode(buffer, tensor);
         }
@@ -51,6 +57,7 @@ public class TypedBinaryFormat {
     public static Tensor decode(Optional<TensorType> type, GrowableByteBuffer buffer) {
         int formatType = buffer.getInt1_4Bytes();
         switch (formatType) {
+            case MIXED_BINARY_FORMAT_TYPE: return new MixedBinaryFormat().decode(type, buffer);
             case SPARSE_BINARY_FORMAT_TYPE: return new SparseBinaryFormat().decode(type, buffer);
             case DENSE_BINARY_FORMAT_TYPE: return new DenseBinaryFormat().decode(type, buffer);
             default: throw new IllegalArgumentException("Binary format type " + formatType + " is unknown");
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java
new file mode 100644
index 00000000000..fef8f05f4e1
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/tensor/MixedTensorTestCase.java
@@ -0,0 +1,155 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.tensor;
+
+import com.google.common.collect.Sets;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Basic mixed tensor tests. Tensor operations are tested in EvaluationTestCase
+ *
+ * @author lesters
+ */
+public class MixedTensorTestCase {
+
+    @Test
+    public void testEmpty() {
+        TensorType type = new TensorType.Builder().mapped("x").indexed("y", 3).build();
+        Tensor empty = Tensor.Builder.of(type).build();
+        assertTrue(empty instanceof MixedTensor);
+        assertTrue(empty.isEmpty());
+        assertEquals("tensor(x{},y[3]):{}", empty.toString());
+        assertEquals("tensor(x{},y[3]):{}", Tensor.from("tensor(x{},y[3]):{}").toString());
+    }
+
+    @Test
+    public void testScalar() {
+        TensorType type = new TensorType.Builder().build();
+        Tensor scalar = MixedTensor.Builder.of(type).cell().value(42.0).build();
+        assertEquals(scalar.asDouble(), 42.0, 1e-6);
+    }
+
+    @Test
+    public void testOneIndexedBuilding() {
+        TensorType type = new TensorType.Builder().indexed("y", 3).build();
+        Tensor tensor = MixedTensor.Builder.of(type).
+                cell().label("y", 0).value(1).
+                cell().label("y", 1).value(2).
+                // {y:2} should be 0.0 and non NaN since we specify indexed size
+                build();
+        assertEquals(Sets.newHashSet("y"), tensor.type().dimensionNames());
+        assertEquals("{{y:0}:1.0,{y:1}:2.0,{y:2}:0.0}",
+                tensor.toString());
+    }
+
+    @Test
+    public void testTwoIndexedBuilding() {
+        TensorType type = new TensorType.Builder().indexed("x").indexed("y", 3).build();
+        Tensor tensor = MixedTensor.Builder.of(type).
+                cell().label("x", 0).label("y", 0).value(1).
+                cell().label("x", 0).label("y", 1).value(2).
+                // {x:1,y:2} should be 0.0 and non NaN since we specify indexed size
+                cell().label("x", 1).label("y", 0).value(4).
+                cell().label("x", 1).label("y", 1).value(5).
+                cell().label("x", 1).label("y", 2).value(6).
+                build();
+        assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames());
+        assertEquals("{{x:0,y:0}:1.0,{x:0,y:1}:2.0,{x:0,y:2}:0.0,{x:1,y:0}:4.0,{x:1,y:1}:5.0,{x:1,y:2}:6.0}",
+                tensor.toString());
+    }
+
+    @Test
+    public void testOneMappedBuilding() {
+        TensorType type = new TensorType.Builder().mapped("x").build();
+        Tensor tensor = MixedTensor.Builder.of(type).
+                cell().label("x", "0").value(1).
+                cell().label("x", "1").value(2).
+                build();
+        assertEquals(Sets.newHashSet("x"), tensor.type().dimensionNames());
+        assertEquals("{{x:0}:1.0,{x:1}:2.0}",
+                tensor.toString());
+    }
+
+    @Test
+    public void testTwoMappedBuilding() {
+        TensorType type = new TensorType.Builder().mapped("x").mapped("y").build();
+        Tensor tensor = MixedTensor.Builder.of(type).
+                cell().label("x", "0").label("y", "0").value(1).
+                cell().label("x", "0").label("y", "1").value(2).
+                cell().label("x", "1").label("y", "0").value(4).
+                cell().label("x", "1").label("y", "1").value(5).
+                cell().label("x", "1").label("y", "2").value(6).
+                build();
+        assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames());
+        assertEquals("{{x:0,y:0}:1.0,{x:0,y:1}:2.0,{x:1,y:0}:4.0,{x:1,y:1}:5.0,{x:1,y:2}:6.0}",
+                tensor.toString());
+    }
+
+    @Test
+    public void testOneMappedOneIndexedBuilding() {
+        TensorType type = new TensorType.Builder().mapped("x").indexed("y", 3).build();
+        Tensor tensor = MixedTensor.Builder.of(type).
+                cell().label("x", "1").label("y", 0).value(1).
+                cell().label("x", "1").label("y", 1).value(2).
+                // {x:1,y:2} should be 0.0 and non NaN since we specify indexed size
+                cell().label("x", "2").label("y", 0).value(4).
+                cell().label("x", "2").label("y", 1).value(5).
+                cell().label("x", "2").label("y", 2).value(6).
+                build();
+        assertEquals(Sets.newHashSet("x", "y"), tensor.type().dimensionNames());
+        assertEquals("{{x:1,y:0}:1.0,{x:1,y:1}:2.0,{x:1,y:2}:0.0,{x:2,y:0}:4.0,{x:2,y:1}:5.0,{x:2,y:2}:6.0}",
+                tensor.toString());
+    }
+
+    @Test
+    public void testTwoMappedOneIndexedBuilding() {
+        TensorType type = new TensorType.Builder().mapped("x").indexed("y").mapped("z").build();
+        Tensor tensor = Tensor.Builder.of(type).
+                cell().label("x", "x1").label("y", 0).label("z","z1").value(1).
+                cell().label("x", "x1").label("y", 0).label("z","z2").value(2).
+                cell().label("x", "x1").label("y", 1).label("z","z1").value(3).
+                cell().label("x", "x1").label("y", 1).label("z","z2").value(4).
+                cell().label("x", "x1").label("y", 2).label("z","z1").value(5).
+                cell().label("x", "x1").label("y", 2).label("z","z2").value(6).
+                cell().label("x", "x2").label("y", 0).label("z","z1").value(11).
+                cell().label("x", "x2").label("y", 0).label("z","z2").value(12).
+                cell().label("x", "x2").label("y", 1).label("z","z1").value(13).
+                cell().label("x", "x2").label("y", 1).label("z","z2").value(14).
+                cell().label("x", "x2").label("y", 2).label("z","z1").value(15).
+                cell().label("x", "x2").label("y", 2).label("z","z2").value(16).
+                build();
+        assertEquals(Sets.newHashSet("x", "y", "z"), tensor.type().dimensionNames());
+        assertEquals("{{x:x1,y:0,z:z1}:1.0,{x:x1,y:0,z:z2}:2.0,{x:x1,y:1,z:z1}:3.0,{x:x1,y:1,z:z2}:4.0,{x:x1,y:2,z:z1}:5.0,{x:x1,y:2,z:z2}:6.0,{x:x2,y:0,z:z1}:11.0,{x:x2,y:0,z:z2}:12.0,{x:x2,y:1,z:z1}:13.0,{x:x2,y:1,z:z2}:14.0,{x:x2,y:2,z:z1}:15.0,{x:x2,y:2,z:z2}:16.0}",
+                tensor.toString());
+    }
+
+    @Test
+    public void testTwoMappedTwoIndexedBuilding() {
+        TensorType type = new TensorType.Builder().mapped("i").indexed("j", 2).mapped("k").indexed("l", 2).build();
+        Tensor tensor = Tensor.Builder.of(type).
+                cell().label("i", "a").label("k","c").label("j",0).label("l",0).value(1).
+                cell().label("i", "a").label("k","c").label("j",0).label("l",1).value(2).
+                cell().label("i", "a").label("k","c").label("j",1).label("l",0).value(3).
+                cell().label("i", "a").label("k","c").label("j",1).label("l",1).value(4).
+                cell().label("i", "a").label("k","d").label("j",0).label("l",0).value(5).
+                cell().label("i", "a").label("k","d").label("j",0).label("l",1).value(6).
+                cell().label("i", "a").label("k","d").label("j",1).label("l",0).value(7).
+                cell().label("i", "a").label("k","d").label("j",1).label("l",1).value(8).
+                cell().label("i", "b").label("k","c").label("j",0).label("l",0).value(9).
+                cell().label("i", "b").label("k","c").label("j",0).label("l",1).value(10).
+                cell().label("i", "b").label("k","c").label("j",1).label("l",0).value(11).
+                cell().label("i", "b").label("k","c").label("j",1).label("l",1).value(12).
+                cell().label("i", "b").label("k","d").label("j",0).label("l",0).value(13).
+                cell().label("i", "b").label("k","d").label("j",0).label("l",1).value(14).
+                cell().label("i", "b").label("k","d").label("j",1).label("l",0).value(15).
+                cell().label("i", "b").label("k","d").label("j",1).label("l",1).value(16).
+                build();
+        assertEquals(Sets.newHashSet("i", "j", "k", "l"), tensor.type().dimensionNames());
+        assertEquals("{{i:a,j:0,k:c,l:0}:1.0,{i:a,j:0,k:c,l:1}:2.0,{i:a,j:0,k:d,l:0}:5.0,{i:a,j:0,k:d,l:1}:6.0,{i:a,j:1,k:c,l:0}:3.0,{i:a,j:1,k:c,l:1}:4.0,{i:a,j:1,k:d,l:0}:7.0,{i:a,j:1,k:d,l:1}:8.0,{i:b,j:0,k:c,l:0}:9.0,{i:b,j:0,k:c,l:1}:10.0,{i:b,j:0,k:d,l:0}:13.0,{i:b,j:0,k:d,l:1}:14.0,{i:b,j:1,k:c,l:0}:11.0,{i:b,j:1,k:c,l:1}:12.0,{i:b,j:1,k:d,l:0}:15.0,{i:b,j:1,k:d,l:1}:16.0}",
+                tensor.toString());
+    }
+
+}
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java
index a653ef97734..7e1f292eb7b 100644
--- a/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java
+++ b/vespajlib/src/test/java/com/yahoo/tensor/functions/ConcatTestCase.java
@@ -43,12 +43,7 @@ public class ConcatTestCase {
         Tensor a = Tensor.from("tensor(x[]):{ {x:0}:1, {x:1}:2 }");
         Tensor b = Tensor.from("tensor(x[]):{ {x:0}:4, {x:1}:5, {x:2}:6 }");
         assertEquals(Tensor.from("tensor(x[5]):{ {x:0}:1, {x:1}:2, {x:2}:4, {x:3}:5, {x:4}:6 }"), a.concat(b, "x"));
-        try {
-            a.concat(b, "y");
-            fail("Expected exception");
-        } catch (IllegalArgumentException expected) {
-            // success
-        }
+        assertEquals(Tensor.from("tensor(x[2],y[2]):{ {x:0,y:0}:1, {x:1,y:0}:2, {x:0,y:1}:4, {x:1,y:1}:5 }"), a.concat(b, "y"));
     }
 
     @Test
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java
new file mode 100644
index 00000000000..b1d7d797b3e
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java
@@ -0,0 +1,95 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.tensor.serialization;
+
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.tensor.MixedTensor;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
+import org.junit.Test;
+
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for the mixed binary format.
+ *
+ * @author lesters
+ */
+public class MixedBinaryFormatTestCase {
+
+    @Test
+    public void testSerialization() {
+        assertSerialization("tensor(x{},y[3]):{{x:1,y:0}:1.0,{x:1,y:1}:2.0,{x:1,y:2}:0.0,{x:2,y:0}:4.0,{x:2,y:1}:5.0,{x:2,y:2}:6.0}");
+        assertSerialization("tensor(x{},y[]):{{x:1,y:0}:1.0,{x:1,y:1}:2.0,{x:1,y:2}:0.0,{x:2,y:0}:4.0,{x:2,y:1}:5.0,{x:2,y:2}:6.0}");
+
+        assertSerialization("tensor(x{},y[3],z{}):{{x:x1,y:0,z:z1}:1.0,{x:x1,y:0,z:z2}:2.0,{x:x1,y:1,z:z1}:3.0,{x:x1,y:1,z:z2}:4.0,{x:x1,y:2,z:z1}:5.0,{x:x1,y:2,z:z2}:6.0,{x:x2,y:0,z:z1}:11.0,{x:x2,y:0,z:z2}:12.0,{x:x2,y:1,z:z1}:13.0,{x:x2,y:1,z:z2}:14.0,{x:x2,y:2,z:z1}:15.0,{x:x2,y:2,z:z2}:16.0}");
+        assertSerialization("tensor(x{},y[],z{}):{{x:x1,y:0,z:z1}:1.0,{x:x1,y:0,z:z2}:2.0,{x:x1,y:1,z:z1}:3.0,{x:x1,y:1,z:z2}:4.0,{x:x1,y:2,z:z1}:5.0,{x:x1,y:2,z:z2}:6.0,{x:x2,y:0,z:z1}:11.0,{x:x2,y:0,z:z2}:12.0,{x:x2,y:1,z:z1}:13.0,{x:x2,y:1,z:z2}:14.0,{x:x2,y:2,z:z1}:15.0,{x:x2,y:2,z:z2}:16.0}");
+
+        assertSerialization("tensor(i{},j[2],k{},l[2]):{{i:a,j:0,k:c,l:0}:1.0,{i:a,j:0,k:c,l:1}:2.0,{i:a,j:0,k:d,l:0}:5.0,{i:a,j:0,k:d,l:1}:6.0,{i:a,j:1,k:c,l:0}:3.0,{i:a,j:1,k:c,l:1}:4.0,{i:a,j:1,k:d,l:0}:7.0,{i:a,j:1,k:d,l:1}:8.0,{i:b,j:0,k:c,l:0}:9.0,{i:b,j:0,k:c,l:1}:10.0,{i:b,j:0,k:d,l:0}:13.0,{i:b,j:0,k:d,l:1}:14.0,{i:b,j:1,k:c,l:0}:11.0,{i:b,j:1,k:c,l:1}:12.0,{i:b,j:1,k:d,l:0}:15.0,{i:b,j:1,k:d,l:1}:16.0}");
+        assertSerialization("tensor(i{},j[],k{},l[]):{{i:a,j:0,k:c,l:0}:1.0,{i:a,j:0,k:c,l:1}:2.0,{i:a,j:0,k:d,l:0}:5.0,{i:a,j:0,k:d,l:1}:6.0,{i:a,j:1,k:c,l:0}:3.0,{i:a,j:1,k:c,l:1}:4.0,{i:a,j:1,k:d,l:0}:7.0,{i:a,j:1,k:d,l:1}:8.0,{i:b,j:0,k:c,l:0}:9.0,{i:b,j:0,k:c,l:1}:10.0,{i:b,j:0,k:d,l:0}:13.0,{i:b,j:0,k:d,l:1}:14.0,{i:b,j:1,k:c,l:0}:11.0,{i:b,j:1,k:c,l:1}:12.0,{i:b,j:1,k:d,l:0}:15.0,{i:b,j:1,k:d,l:1}:16.0}");
+    }
+
+    @Test
+    public void testOneIndexedSerialization() {
+        TensorType type = new TensorType.Builder().indexed("y", 3).build();
+        Tensor tensor = MixedTensor.Builder.of(type).
+                cell().label("y", 0).value(1).
+                cell().label("y", 1).value(2).
+                build();
+        assertSerialization(tensor);
+    }
+
+    @Test
+    public void testTwoIndexedSerialization() {
+        TensorType type = new TensorType.Builder().indexed("x").indexed("y", 3).build();
+        Tensor tensor = MixedTensor.Builder.of(type).
+                cell().label("x", 0).label("y", 0).value(1).
+                cell().label("x", 0).label("y", 1).value(2).
+                cell().label("x", 1).label("y", 0).value(4).
+                cell().label("x", 1).label("y", 1).value(5).
+                cell().label("x", 1).label("y", 2).value(6).
+                build();
+        assertSerialization(tensor);
+    }
+
+    @Test
+    public void testOneMappedSerialization() {
+        TensorType type = new TensorType.Builder().mapped("x").build();
+        Tensor tensor = MixedTensor.Builder.of(type).
+                cell().label("x", "0").value(1).
+                cell().label("x", "1").value(2).
+                build();
+        assertSerialization(tensor);
+    }
+
+    @Test
+    public void testTwoMappedSerialization() {
+        TensorType type = new TensorType.Builder().mapped("x").mapped("y").build();
+        Tensor tensor = MixedTensor.Builder.of(type).
+                cell().label("x", "0").label("y", "0").value(1).
+                cell().label("x", "0").label("y", "1").value(2).
+                cell().label("x", "1").label("y", "0").value(4).
+                cell().label("x", "1").label("y", "1").value(5).
+                cell().label("x", "1").label("y", "2").value(6).
+                build();
+        assertSerialization(tensor);
+    }
+
+    private void assertSerialization(String tensorString) {
+        assertSerialization(Tensor.from(tensorString));
+    }
+
+    private void assertSerialization(Tensor tensor) {
+        assertSerialization(tensor, tensor.type());
+    }
+    
+    private void assertSerialization(Tensor tensor, TensorType expectedType) {
+        byte[] encodedTensor = TypedBinaryFormat.encode(tensor);
+        Tensor decodedTensor = TypedBinaryFormat.decode(Optional.of(expectedType), GrowableByteBuffer.wrap(encodedTensor));
+        assertEquals(tensor, decodedTensor);
+    }
+
+}
+
diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/SerializationTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SerializationTestCase.java
new file mode 100644
index 00000000000..68bf59e3ed9
--- /dev/null
+++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SerializationTestCase.java
@@ -0,0 +1,149 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.tensor.serialization;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.yahoo.io.GrowableByteBuffer;
+import com.yahoo.tensor.Tensor;
+import com.yahoo.tensor.TensorType;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class SerializationTestCase {
+
+    private static String testPath = "eval/src/apps/make_tensor_binary_format_test_spec/test_spec.json";
+    private static List<String> tests = new ArrayList<>();
+
+    @Before
+    public void loadTests() throws IOException {
+        File testSpec = new File(testPath);
+        if (!testSpec.exists()) {
+            testSpec = new File("../" + testPath);
+        }
+        try(BufferedReader br = new BufferedReader(new FileReader(testSpec))) {
+            String test = br.readLine();
+            while (test != null) {
+                tests.add(test);
+                test = br.readLine();
+            }
+        }
+    }
+
+    @Test
+    public void testSerialization() throws IOException {
+        for (String test : tests) {
+            ObjectMapper mapper = new ObjectMapper();
+            JsonNode node = mapper.readTree(test);
+            if (node.has("tensor") && node.has("binary")) {
+                System.out.println("Running test: " + test);
+                
+                Tensor tensor = buildTensor(node.get("tensor"));
+                String spec = getSpec(node.get("tensor"));
+                byte[] encodedTensor = TypedBinaryFormat.encode(tensor);
+                boolean serializedToABinaryRepresentation = false;
+
+                JsonNode binaryNode = node.get("binary");
+                for (int i = 0; i < binaryNode.size(); ++i) {
+                    byte[] bin = getBytes(binaryNode.get(i).asText());
+                    Tensor decodedTensor = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(bin));
+
+                    if (spec.equalsIgnoreCase("double")) {
+                        assertEquals(tensor.asDouble(), decodedTensor.asDouble(), 1e-6);
+                    } else {
+                        assertEquals(tensor, decodedTensor);
+                    }
+
+                    if (Arrays.equals(encodedTensor, bin)) {
+                        serializedToABinaryRepresentation = true;
+                    }
+                }
+                assertTrue("Tensor did not serialize to one of the given representations", serializedToABinaryRepresentation);
+            }
+        }
+    }
+
+    private Tensor buildTensor(JsonNode tensor) {
+        TensorType type = tensorType(tensor);
+        Tensor.Builder builder = Tensor.Builder.of(type);
+        tensorCells(tensor, builder);
+        return builder.build();
+    }
+
+    private TensorType tensorType(JsonNode tensor) {
+        String spec = getSpec(tensor);
+        if (spec.equalsIgnoreCase("double")) {
+            spec = "tensor()";
+        }
+        return TensorType.fromSpec(spec);
+    }
+
+    private String getSpec(JsonNode tensor) {
+        return tensor.get("type").asText();
+    }
+
+    private void tensorCells(JsonNode tensor, Tensor.Builder builder) {
+        JsonNode cells = tensor.get("cells");
+        for (JsonNode cell : cells) {
+            tensorCell(cell, builder.cell());
+        }
+    }
+
+    private void tensorCell(JsonNode cell, Tensor.Builder.CellBuilder cellBuilder) {
+        tensorCellAddress(cellBuilder, cell.get("address"));
+        tensorCellValue(cellBuilder, cell.get("value"));
+    }
+
+    private void tensorCellValue(Tensor.Builder.CellBuilder cellBuilder, JsonNode value) {
+        cellBuilder.value(value.doubleValue());
+    }
+
+    private void tensorCellAddress(Tensor.Builder.CellBuilder cellBuilder, JsonNode address) {
+        Iterator<String> dimension = address.fieldNames();
+        while (dimension.hasNext()) {
+            String name = dimension.next();
+            JsonNode label = address.get(name);
+            cellBuilder.label(name, label.asText());
+        }
+    }
+
+    private byte[] getBytes(String binaryRepresentation) {
+        return parseHexValue(binaryRepresentation.substring(2));
+    }
+    
+    private byte[] parseHexValue(String s) {
+        final int len = s.length();
+        byte[] bytes = new byte[len/2];
+        for (int i = 0; i < len; i += 2) {
+            int c1 = hexValue(s.charAt(i)) << 4;
+            int c2 = hexValue(s.charAt(i + 1));
+            bytes[i/2] = (byte)(c1 + c2);
+        }
+        return bytes;
+    }
+
+    private int hexValue(Character c) {
+        if (c >= 'a' && c <= 'f') {
+            return c - 'a' + 10;
+        } else if (c >= 'A' && c <= 'F') {
+            return c - 'A' + 10;
+        } else if (c >= '0' && c <= '9') {
+            return c - '0';
+        }
+        throw new IllegalArgumentException("Hex contains illegal characters");
+    }
+
+}