add new_encode and new_decode

* move common encode/decode logic from
  simple_tensor.cpp into codec.h / codec.cpp
* encoding and decoding using new Value API
  is in value_codec.h, both for binary format
  and to/from TensorSpec
* will move implementation of TensorSpec
  conversions later
* extend unit test for tensor serialization
  to also test new_encode and new_decode
* add hack in CreateValueFromTensorSpec to
  handle a spec without any cells but
  no mapped dimensions, this was used by
  unit test

11 files changed, 339 insertions, 145 deletions

diff --git a/eval/src/tests/eval/simple_value/simple_value_test.cpp b/eval/src/tests/eval/simple_value/simple_value_test.cpp
index 8ac553680f7..233c09f7e2b 100644
--- a/eval/src/tests/eval/simple_value/simple_value_test.cpp
+++ b/eval/src/tests/eval/simple_value/simple_value_test.cpp
@@ -1,6 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/eval/eval/simple_value.h>
+#include <vespa/eval/eval/value_codec.h>
 #include <vespa/eval/eval/test/tensor_model.hpp>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/gtest/gtest.h>
diff --git a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp
index 48c72c3c591..eb7607c13b8 100644
--- a/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp
+++ b/eval/src/tests/tensor/packed_mappings/packed_mixed_test.cpp
@@ -1,6 +1,7 @@
 // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/eval/eval/simple_value.h>
+#include <vespa/eval/eval/value_codec.h>
 #include <vespa/eval/eval/test/tensor_model.hpp>
 #include <vespa/eval/tensor/mixed/packed_mixed_factory.h>
 #include <vespa/vespalib/gtest/gtest.h>
diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
index d1491e4f758..c00b0e5d35b 100644
--- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
+++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp
@@ -10,6 +10,7 @@
 #include <vespa/vespalib/objects/hexdump.h>
 #include <ostream>
 #include <vespa/eval/tensor/dense/dense_tensor_view.h>
+#include <vespa/eval/eval/value_codec.h>
 
 using namespace vespalib::tensor;
 using vespalib::eval::TensorSpec;
@@ -47,6 +48,24 @@ void verify_cells_only(const ExpBuffer &exp, const TensorSpec &spec) {
     ASSERT_EQUAL(i, cells.size());
 }
 
+TensorSpec verify_new_value_serialized(const ExpBuffer &exp, const TensorSpec &spec) {
+    auto factory = vespalib::eval::SimpleValueBuilderFactory();
+    auto new_value = vespalib::eval::value_from_spec(spec, factory);
+    auto new_value_spec = vespalib::eval::spec_from_value(*new_value);
+    nbostream actual;
+    vespalib::eval::new_encode(*new_value, actual);
+    ASSERT_EQUAL(exp, actual);
+    auto new_decoded = vespalib::eval::new_decode(actual, factory);
+    auto new_decoded_spec = vespalib::eval::spec_from_value(*new_decoded);
+    EXPECT_EQUAL(0u, actual.size());
+    EXPECT_EQUAL(new_value_spec, new_decoded_spec);
+    if (new_value->type().is_dense()) {
+        TEST_DO(verify_cells_only<float>(exp, new_value_spec));
+        TEST_DO(verify_cells_only<double>(exp, new_value_spec));
+    }
+    return new_decoded_spec;
+}
+
 void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) {
     auto &engine = DefaultTensorEngine::ref();
     auto value = engine.from_spec(spec);
@@ -62,6 +81,8 @@ void verify_serialized(const ExpBuffer &exp, const TensorSpec &spec) {
         TEST_DO(verify_cells_only<float>(exp, value_spec));
         TEST_DO(verify_cells_only<double>(exp, value_spec));
     }
+    auto new_value_spec = verify_new_value_serialized(exp, spec);
+    EXPECT_EQUAL(value_spec, new_value_spec);
 }
 
 //-----------------------------------------------------------------------------
diff --git a/eval/src/vespa/eval/eval/CMakeLists.txt b/eval/src/vespa/eval/eval/CMakeLists.txt
index 973245607de..4660a2ac9bf 100644
--- a/eval/src/vespa/eval/eval/CMakeLists.txt
+++ b/eval/src/vespa/eval/eval/CMakeLists.txt
@@ -4,6 +4,7 @@ vespa_add_library(eval_eval OBJECT
     aggr.cpp
     basic_nodes.cpp
     call_nodes.cpp
+    codec.cpp
     compile_tensor_function.cpp
     delete_node.cpp
     fast_forest.cpp
@@ -28,6 +29,7 @@ vespa_add_library(eval_eval OBJECT
     tensor_nodes.cpp
     tensor_spec.cpp
     value.cpp
+    value_codec.cpp
     value_type.cpp
     value_type_spec.cpp
     visit_stuff.cpp
diff --git a/eval/src/vespa/eval/eval/codec.cpp b/eval/src/vespa/eval/eval/codec.cpp
new file mode 100644
index 00000000000..1cf60f476a1
--- /dev/null
+++ b/eval/src/vespa/eval/eval/codec.cpp
@@ -0,0 +1,46 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "codec.h"
+
+namespace vespalib::eval::codec {
+
+void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) {
+    maybe_encode_cell_type(output, format, meta);
+    if (format.is_sparse) {
+        output.putInt1_4Bytes(meta.mapped.size());
+        for (size_t idx: meta.mapped) {
+            output.writeSmallString(type.dimensions()[idx].name);
+        }
+    }
+    if (format.is_dense) {
+        output.putInt1_4Bytes(meta.indexed.size());
+        for (size_t idx: meta.indexed) {
+            output.writeSmallString(type.dimensions()[idx].name);
+            output.putInt1_4Bytes(type.dimensions()[idx].size);
+        }
+    }
+}
+
+ValueType decode_type(nbostream &input, const Format &format) {
+    CellType cell_type = maybe_decode_cell_type(input, format);
+    std::vector<ValueType::Dimension> dim_list;
+    if (format.is_sparse) {
+        size_t cnt = input.getInt1_4Bytes();
+        for (size_t i = 0; i < cnt; ++i) {
+            vespalib::string name;
+            input.readSmallString(name);
+            dim_list.emplace_back(name);
+        }
+    }
+    if (format.is_dense) {
+        size_t cnt = input.getInt1_4Bytes();
+        for (size_t i = 0; i < cnt; ++i) {
+            vespalib::string name;
+            input.readSmallString(name);
+            dim_list.emplace_back(name, input.getInt1_4Bytes());
+        }
+    }
+    return ValueType::tensor_type(std::move(dim_list), cell_type);
+}
+
+} // namespace vespalib::eval::codec
diff --git a/eval/src/vespa/eval/eval/codec.h b/eval/src/vespa/eval/eval/codec.h
new file mode 100644
index 00000000000..fad370f6d8b
--- /dev/null
+++ b/eval/src/vespa/eval/eval/codec.h
@@ -0,0 +1,111 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "simple_tensor.h"
+#include <vespa/vespalib/objects/nbostream.h>
+#include <cassert>
+
+namespace vespalib::eval::codec {
+
+using CellType = ValueType::CellType;
+using IndexList = std::vector<size_t>;
+
+constexpr uint32_t DOUBLE_CELL_TYPE = 0;
+constexpr uint32_t FLOAT_CELL_TYPE = 1;
+
+inline uint32_t cell_type_to_id(CellType cell_type) {
+    switch (cell_type) {
+    case CellType::DOUBLE: return DOUBLE_CELL_TYPE;
+    case CellType::FLOAT: return FLOAT_CELL_TYPE;
+    }
+    abort();
+}
+
+inline CellType id_to_cell_type(uint32_t id) {
+    switch (id) {
+    case DOUBLE_CELL_TYPE: return CellType::DOUBLE;
+    case FLOAT_CELL_TYPE: return CellType::FLOAT;
+    }
+    abort();
+}
+
+/**
+ * Meta information about how a type can be decomposed into mapped and
+ * indexed dimensions and also how large each block is. A block is a
+ * dense-subspace consisting of all indexed dimensions that is
+ * uniquely specified by the labels of all mapped dimensions.
+ **/
+struct TypeMeta {
+    IndexList mapped;
+    IndexList indexed;
+    size_t block_size;
+    CellType cell_type;
+    explicit TypeMeta(const ValueType &type)
+        : mapped(),
+          indexed(),
+          block_size(1),
+          cell_type(type.cell_type())
+    {
+        for (size_t i = 0; i < type.dimensions().size(); ++i) {
+            const auto &dimension = type.dimensions()[i];
+            if (dimension.is_mapped()) {
+                mapped.push_back(i);
+            } else {
+                block_size *= dimension.size;
+                indexed.push_back(i);
+            }
+        }
+    }
+    ~TypeMeta() {}
+};
+
+struct Format {
+    bool     is_sparse;
+    bool     is_dense;
+    bool     with_cell_type;
+    uint32_t tag;
+    explicit Format(const TypeMeta &meta)
+        : is_sparse(meta.mapped.size() > 0),
+          is_dense((meta.indexed.size() > 0) || !is_sparse),
+          with_cell_type(meta.cell_type != CellType::DOUBLE),
+          tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {}
+    explicit Format(uint32_t tag_in)
+        : is_sparse((tag_in & 0x1) != 0),
+          is_dense((tag_in & 0x2) != 0),
+          with_cell_type((tag_in & 0x4) != 0),
+          tag(tag_in) {}
+    ~Format() {}
+};
+
+inline void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeMeta &meta) {
+    if (format.with_cell_type) {
+        output.putInt1_4Bytes(cell_type_to_id(meta.cell_type));
+    }
+}
+
+void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta);
+
+inline void maybe_encode_num_blocks(nbostream &output, const TypeMeta &meta, size_t num_blocks) {
+    if ((meta.mapped.size() > 0)) {
+        output.putInt1_4Bytes(num_blocks);
+    }
+}
+
+inline CellType maybe_decode_cell_type(nbostream &input, const Format &format) {
+    if (format.with_cell_type) {
+        return id_to_cell_type(input.getInt1_4Bytes());
+    }
+    return CellType::DOUBLE;
+}
+
+ValueType decode_type(nbostream &input, const Format &format);
+
+inline size_t maybe_decode_num_blocks(nbostream &input, const TypeMeta &meta, const Format &format) {
+    if ((meta.mapped.size() > 0) || !format.is_dense) {
+        return input.getInt1_4Bytes();
+    }
+    return 1;
+}
+
+} // namespace vespalib::eval::codec
diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp
index 64b2b6f8865..498119f0733 100644
--- a/eval/src/vespa/eval/eval/simple_tensor.cpp
+++ b/eval/src/vespa/eval/eval/simple_tensor.cpp
@@ -3,44 +3,26 @@
 #include "simple_tensor.h"
 #include "simple_tensor_engine.h"
 #include "operation.h"
+#include "codec.h"
 #include <vespa/vespalib/util/overload.h>
 #include <vespa/vespalib/util/visit_ranges.h>
 #include <vespa/vespalib/objects/nbostream.h>
 #include <algorithm>
 #include <cassert>
 
+using namespace vespalib::eval::codec;
+
 namespace vespalib {
 namespace eval {
 
 using Address = SimpleTensor::Address;
 using Cell = SimpleTensor::Cell;
 using Cells = SimpleTensor::Cells;
-using IndexList = std::vector<size_t>;
 using Label = SimpleTensor::Label;
 using CellRef = std::reference_wrapper<const Cell>;
-using CellType = ValueType::CellType;
 
 namespace {
 
-constexpr uint32_t DOUBLE_CELL_TYPE = 0;
-constexpr uint32_t FLOAT_CELL_TYPE = 1;
-
-uint32_t cell_type_to_id(CellType cell_type) {
-    switch (cell_type) {
-    case CellType::DOUBLE: return DOUBLE_CELL_TYPE;
-    case CellType::FLOAT: return FLOAT_CELL_TYPE;
-    }
-    abort();
-}
-
-CellType id_to_cell_type(uint32_t id) {
-    switch (id) {
-    case DOUBLE_CELL_TYPE: return CellType::DOUBLE;
-    case FLOAT_CELL_TYPE: return CellType::FLOAT;
-    }
-    abort();
-}
-
 void assert_type(const ValueType &type) {
     (void) type;
     assert(type.is_double() || type.is_tensor());
@@ -105,35 +87,6 @@ const vespalib::string &reverse_rename(const vespalib::string &name,
     return name;
 }
 
-/**
- * Meta information about how a type can be decomposed into mapped and
- * indexed dimensions and also how large each block is. A block is a
- * dense-subspace consisting of all indexed dimensions that is
- * uniquely specified by the labels of all mapped dimensions.
- **/
-struct TypeMeta {
-    IndexList mapped;
-    IndexList indexed;
-    size_t block_size;
-    CellType cell_type;
-    explicit TypeMeta(const ValueType &type)
-        : mapped(),
-          indexed(),
-          block_size(1),
-          cell_type(type.cell_type())
-    {
-        for (size_t i = 0; i < type.dimensions().size(); ++i) {
-            const auto &dimension = type.dimensions()[i];
-            if (dimension.is_mapped()) {
-                mapped.push_back(i);
-            } else {
-                block_size *= dimension.size;
-                indexed.push_back(i);
-            }
-        }
-    }
-    ~TypeMeta() {}
-};
 
 /**
  * Helper class used when building SimpleTensors. While a tensor
@@ -438,95 +391,12 @@ public:
     }
 };
 
-struct Format {
-    bool     is_sparse;
-    bool     is_dense;
-    bool     with_cell_type;
-    uint32_t tag;
-    explicit Format(const TypeMeta &meta)
-        : is_sparse(meta.mapped.size() > 0),
-          is_dense((meta.indexed.size() > 0) || !is_sparse),
-          with_cell_type(meta.cell_type != CellType::DOUBLE),
-          tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0) | (with_cell_type ? 0x4 : 0)) {}
-    explicit Format(uint32_t tag_in)
-        : is_sparse((tag_in & 0x1) != 0),
-          is_dense((tag_in & 0x2) != 0),
-          with_cell_type((tag_in & 0x4) != 0),
-          tag(tag_in) {}
-    ~Format() {}
-};
-
-void maybe_encode_cell_type(nbostream &output, const Format &format, const TypeMeta &meta) {
-    if (format.with_cell_type) {
-        output.putInt1_4Bytes(cell_type_to_id(meta.cell_type));
-    }
-}
-
-void encode_type(nbostream &output, const Format &format, const ValueType &type, const TypeMeta &meta) {
-    maybe_encode_cell_type(output, format, meta);
-    if (format.is_sparse) {
-        output.putInt1_4Bytes(meta.mapped.size());
-        for (size_t idx: meta.mapped) {
-            output.writeSmallString(type.dimensions()[idx].name);
-        }
-    }
-    if (format.is_dense) {
-        output.putInt1_4Bytes(meta.indexed.size());
-        for (size_t idx: meta.indexed) {
-            output.writeSmallString(type.dimensions()[idx].name);
-            output.putInt1_4Bytes(type.dimensions()[idx].size);
-        }
-    }
-}
-
-void maybe_encode_num_blocks(nbostream &output, const TypeMeta &meta, size_t num_blocks) {
-    if ((meta.mapped.size() > 0)) {
-        output.putInt1_4Bytes(num_blocks);
-    }
-}
-
 void encode_mapped_labels(nbostream &output, const TypeMeta &meta, const Address &addr) {
     for (size_t idx: meta.mapped) {
         output.writeSmallString(addr[idx].name);
     }
 }
 
-CellType maybe_decode_cell_type(nbostream &input, const Format &format) {
-    if (format.with_cell_type) {
-        return id_to_cell_type(input.getInt1_4Bytes());
-    }
-    return CellType::DOUBLE;
-}
-
-ValueType decode_type(nbostream &input, const Format &format) {
-    CellType cell_type = maybe_decode_cell_type(input, format);
-    std::vector<ValueType::Dimension> dim_list;
-    if (format.is_sparse) {
-        size_t cnt = input.getInt1_4Bytes();
-        for (size_t i = 0; i < cnt; ++i) {
-            vespalib::string name;
-            input.readSmallString(name);
-            dim_list.emplace_back(name);
-        }
-    }
-    if (format.is_dense) {
-        size_t cnt = input.getInt1_4Bytes();
-        for (size_t i = 0; i < cnt; ++i) {
-            vespalib::string name;
-            input.readSmallString(name);
-            dim_list.emplace_back(name, input.getInt1_4Bytes());
-        }
-    }
-    return ValueType::tensor_type(std::move(dim_list), cell_type);
-}
-
-size_t maybe_decode_num_blocks(nbostream &input, const TypeMeta &meta, const Format &format) {
-    if ((meta.mapped.size() > 0) || !format.is_dense) {
-        return input.getInt1_4Bytes();
-    }
-    return 1;
-}
-
 void decode_mapped_labels(nbostream &input, const TypeMeta &meta, Address &addr) {
     for (size_t idx: meta.mapped) {
         vespalib::string name;
diff --git a/eval/src/vespa/eval/eval/simple_value.cpp b/eval/src/vespa/eval/eval/simple_value.cpp
index 4daa78375e5..2c958f2150a 100644
--- a/eval/src/vespa/eval/eval/simple_value.cpp
+++ b/eval/src/vespa/eval/eval/simple_value.cpp
@@ -3,6 +3,7 @@
 #include "simple_value.h"
 #include "tensor_spec.h"
 #include "inline_operation.h"
+#include "codec.h"
 #include <vespa/vespalib/util/typify.h>
 #include <vespa/vespalib/util/visit_ranges.h>
 #include <vespa/vespalib/util/overload.h>
@@ -45,6 +46,10 @@ struct CreateValueFromTensorSpec {
             }
             map[sparse_key][dense_key] = entry.second;
         }
+        // hack for passing some (invalid?) unit tests
+        if (spec.cells().empty() && type.count_mapped_dimensions() == 0) {
+            map[SparseKey()][0] = 0;
+        }
         auto builder = factory.create_value_builder<T>(type, type.count_mapped_dimensions(), type.dense_subspace_size(), map.size());
         for (const auto &entry: map) {
             auto subspace = builder->add_subspace(entry.first);
@@ -410,4 +415,4 @@ TensorSpec spec_from_value(const Value &value) {
 
 //-----------------------------------------------------------------------------
 
-}
+} // namespace
diff --git a/eval/src/vespa/eval/eval/simple_value.h b/eval/src/vespa/eval/eval/simple_value.h
index 4c7bccfa01c..14ba5df835e 100644
--- a/eval/src/vespa/eval/eval/simple_value.h
+++ b/eval/src/vespa/eval/eval/simple_value.h
@@ -203,15 +203,4 @@ struct SparseJoinPlan {
 using join_fun_t = double (*)(double, double);
 std::unique_ptr<Value> new_join(const Value &a, const Value &b, join_fun_t function, const ValueBuilderFactory &factory);
 
-/**
- * Make a value from a tensor spec using a value builder factory
- * interface, making it work with any value implementation.
- **/
-std::unique_ptr<Value> value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory);
-
-/**
- * Convert a generic value to a tensor spec.
- **/
-TensorSpec spec_from_value(const Value &value);
-
 }
diff --git a/eval/src/vespa/eval/eval/value_codec.cpp b/eval/src/vespa/eval/eval/value_codec.cpp
new file mode 100644
index 00000000000..5aad3f91b6a
--- /dev/null
+++ b/eval/src/vespa/eval/eval/value_codec.cpp
@@ -0,0 +1,115 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "value_codec.h"
+#include "codec.h"
+#include <vespa/vespalib/util/typify.h>
+
+using namespace vespalib::eval::codec;
+
+namespace vespalib::eval {
+
+void encode_mapped_labels(nbostream &output, size_t num_mapped_dims, const std::vector<vespalib::stringref> &addr) {
+    for (size_t i = 0; i < num_mapped_dims; ++i) {
+        output.writeSmallString(addr[i]);
+    }
+}
+
+void decode_mapped_labels(nbostream &input, size_t num_mapped_dims, std::vector<vespalib::string> &addr) {
+    for (size_t i = 0; i < num_mapped_dims; ++i) {
+        vespalib::string name;
+        input.readSmallString(name);
+        addr[i] = name;
+    }
+}
+
+void new_encode(const Value &value, nbostream &output) {
+    TypeMeta meta(value.type());
+    Format format(meta);
+    output.putInt1_4Bytes(format.tag);
+    encode_type(output, format, value.type(), meta);
+    maybe_encode_num_blocks(output, meta, value.cells().size / meta.block_size);
+    std::vector<vespalib::stringref> address(meta.mapped.size());
+    std::vector<vespalib::stringref*> a_refs(meta.mapped.size());
+    for (size_t i = 0; i < meta.mapped.size(); ++i) {
+        a_refs[i] = &address[i];
+    }
+    auto view = value.index().create_view({});
+    view->lookup({});
+    size_t subspace;
+    while (view->next_result(a_refs, subspace)) {
+        encode_mapped_labels(output, meta.mapped.size(), address);
+        if (meta.cell_type == CellType::FLOAT) {
+            auto iter = value.cells().typify<float>().begin();
+            iter += (subspace * meta.block_size);
+            for (size_t i = 0; i < meta.block_size; ++i) {
+                output << (float) *iter++;
+            }
+        } else {
+            auto iter = value.cells().typify<double>().begin();
+            iter += (subspace * meta.block_size);
+            for (size_t i = 0; i < meta.block_size; ++i) {
+                output << *iter++;
+            }
+        }
+    }
+}
+
+template<typename T>
+void decode_cells(nbostream &input, size_t num_cells, ArrayRef<T> dst)
+{
+    T value;
+    for (size_t i = 0; i < num_cells; ++i) {
+        input >> value;
+        dst[i] = value;
+    }
+}
+
+struct DecodeState {
+    const ValueType &type;
+    const size_t block_size;
+    const size_t num_blocks;
+    const size_t num_mapped_dims;
+    std::vector<vespalib::string> address;
+    std::vector<vespalib::stringref> addr_refs;
+    DecodeState(const ValueType &type_in,
+                size_t block_size_in,
+                size_t num_blocks_in,
+                size_t num_mapped_dims_in)
+      : type(type_in),
+        block_size(block_size_in),
+        num_blocks(num_blocks_in),
+        num_mapped_dims(num_mapped_dims_in),
+        address(num_mapped_dims_in),
+        addr_refs(num_mapped_dims_in)
+    {}
+    void fix_refs() {
+        for (size_t j = 0; j < num_mapped_dims; ++j) {
+            addr_refs[j] = address[j];
+        }
+    }
+};
+
+struct ContentDecoder {
+    template<typename T>
+    static std::unique_ptr<Value> invoke(nbostream &input, DecodeState &state, const ValueBuilderFactory &factory) {
+        auto builder = factory.create_value_builder<T>(state.type, state.num_mapped_dims, state.block_size, state.num_blocks);
+        for (size_t i = 0; i < state.num_blocks; ++i) {
+            decode_mapped_labels(input, state.num_mapped_dims, state.address);
+            state.fix_refs();
+            auto block_cells = builder->add_subspace(state.addr_refs);
+            decode_cells(input, state.block_size, block_cells);
+        }
+        return builder->build(std::move(builder));
+    }
+};
+
+std::unique_ptr<Value> new_decode(nbostream &input, const ValueBuilderFactory &factory) {
+    Format format(input.getInt1_4Bytes());
+    ValueType type = decode_type(input, format);
+    TypeMeta meta(type);
+    const size_t num_blocks = maybe_decode_num_blocks(input, meta, format);
+    DecodeState state(type, meta.block_size, num_blocks, meta.mapped.size());
+    return typify_invoke<1,TypifyCellType,ContentDecoder>(meta.cell_type, input, state, factory);
+}
+
+} // namespace
diff --git a/eval/src/vespa/eval/eval/value_codec.h b/eval/src/vespa/eval/eval/value_codec.h
new file mode 100644
index 00000000000..08b0ac73cc3
--- /dev/null
+++ b/eval/src/vespa/eval/eval/value_codec.h
@@ -0,0 +1,33 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include "simple_value.h"
+#include <vespa/vespalib/stllike/string.h>
+
+namespace vespalib { class nbostream; }
+
+namespace vespalib::eval {
+
+/**
+ * encode a value to binary format
+ **/
+void new_encode(const Value &value, nbostream &output);
+
+/**
+ * decode a value from binary format
+ **/
+std::unique_ptr<Value> new_decode(nbostream &input, const ValueBuilderFactory &factory);
+
+/**
+ * Make a value from a tensor spec using a value builder factory
+ * interface, making it work with any value implementation.
+ **/
+std::unique_ptr<Value> value_from_spec(const TensorSpec &spec, const ValueBuilderFactory &factory);
+
+/**
+ * Convert a generic value to a tensor spec.
+ **/
+TensorSpec spec_from_value(const Value &value);
+
+}