diff options
4 files changed, 13 insertions, 9 deletions
diff --git a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp index b4e0db4fa8f..b7aa988775d 100644 --- a/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp +++ b/eval/src/tests/tensor/tensor_serialization/tensor_serialization_test.cpp @@ -265,7 +265,7 @@ TEST_F("test tensor serialization for DenseTensor", DenseFixture) { } TEST_F("test 'float' cells", DenseFixture) { - TEST_DO(f.assertSerialized({0x04, 0x01, 0x02, 0x01, 0x78, 0x03, + TEST_DO(f.assertSerialized({0x06, 0x01, 0x02, 0x01, 0x78, 0x03, 0x01, 0x79, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, diff --git a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp index 813763ba268..4ca037e82a4 100644 --- a/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp +++ b/eval/src/vespa/eval/tensor/serialization/typed_binary_format.cpp @@ -25,7 +25,9 @@ namespace { constexpr uint32_t SPARSE_BINARY_FORMAT_TYPE = 1u; constexpr uint32_t DENSE_BINARY_FORMAT_TYPE = 2u; constexpr uint32_t MIXED_BINARY_FORMAT_TYPE = 3u; -constexpr uint32_t TYPED_DENSE_BINARY_FORMAT_TYPE = 4u; +constexpr uint32_t SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5u; //Future +constexpr uint32_t DENSE_BINARY_FORMAT_WITH_CELLTYPE = 6u; +constexpr uint32_t MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7u; //Future constexpr uint32_t DOUBLE_VALUE_TYPE = 0; constexpr uint32_t FLOAT_VALUE_TYPE = 1; @@ -60,7 +62,7 @@ TypedBinaryFormat::serialize(nbostream &stream, const Tensor &tensor, SerializeF { if (auto denseTensor = dynamic_cast<const DenseTensorView *>(&tensor)) { if (format != SerializeFormat::DOUBLE) { - stream.putInt1_4Bytes(TYPED_DENSE_BINARY_FORMAT_TYPE); + stream.putInt1_4Bytes(DENSE_BINARY_FORMAT_WITH_CELLTYPE); stream.putInt1_4Bytes(format2Encoding(format)); DenseBinaryFormat(format).serialize(stream, *denseTensor); } else { @@ -89,7 +91,7 @@ TypedBinaryFormat::deserialize(nbostream &stream) if (formatId == DENSE_BINARY_FORMAT_TYPE) { return DenseBinaryFormat(SerializeFormat::DOUBLE).deserialize(stream); } - if (formatId == TYPED_DENSE_BINARY_FORMAT_TYPE) { + if (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) { return DenseBinaryFormat(encoding2Format(stream.getInt1_4Bytes())).deserialize(stream); } if (formatId == MIXED_BINARY_FORMAT_TYPE) { @@ -107,7 +109,7 @@ TypedBinaryFormat::deserializeCellsOnlyFromDenseTensors(nbostream &stream, std:: if (formatId == DENSE_BINARY_FORMAT_TYPE) { return DenseBinaryFormat(SerializeFormat::DOUBLE).deserializeCellsOnly(stream, cells); } - if (formatId == TYPED_DENSE_BINARY_FORMAT_TYPE) { + if (formatId == DENSE_BINARY_FORMAT_WITH_CELLTYPE) { return DenseBinaryFormat(encoding2Format(stream.getInt1_4Bytes())).deserializeCellsOnly(stream, cells); } abort(); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java index f2c5d4e2bd8..9b298f1dffb 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java @@ -23,7 +23,9 @@ public class TypedBinaryFormat { private static final int SPARSE_BINARY_FORMAT_TYPE = 1; private static final int DENSE_BINARY_FORMAT_TYPE = 2; private static final int MIXED_BINARY_FORMAT_TYPE = 3; - private static final int TYPED_DENSE_BINARY_FORMAT_TYPE = 4; + private static final int SPARSE_BINARY_FORMAT_WITH_CELLTYPE = 5; + private static final int DENSE_BINARY_FORMAT_WITH_CELLTYPE = 6; + private static final int MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7; public static byte[] encode(Tensor tensor) { GrowableByteBuffer buffer = new GrowableByteBuffer(); @@ -38,7 +40,7 @@ public class TypedBinaryFormat { new DenseBinaryFormat(DenseBinaryFormat.EncodeType.DOUBLE_IS_DEFAULT).encode(buffer, tensor); break; default: - buffer.putInt1_4Bytes(TYPED_DENSE_BINARY_FORMAT_TYPE); + buffer.putInt1_4Bytes(DENSE_BINARY_FORMAT_WITH_CELLTYPE); new DenseBinaryFormat(DenseBinaryFormat.EncodeType.NO_DEFAULT).encode(buffer, tensor); break; } @@ -67,7 +69,7 @@ public class TypedBinaryFormat { case MIXED_BINARY_FORMAT_TYPE: return new MixedBinaryFormat().decode(type, buffer); case SPARSE_BINARY_FORMAT_TYPE: return new SparseBinaryFormat().decode(type, buffer); case DENSE_BINARY_FORMAT_TYPE: return new DenseBinaryFormat(DenseBinaryFormat.EncodeType.DOUBLE_IS_DEFAULT).decode(type, buffer); - case TYPED_DENSE_BINARY_FORMAT_TYPE: return new DenseBinaryFormat(DenseBinaryFormat.EncodeType.NO_DEFAULT).decode(type, buffer); + case DENSE_BINARY_FORMAT_WITH_CELLTYPE: return new DenseBinaryFormat(DenseBinaryFormat.EncodeType.NO_DEFAULT).decode(type, buffer); default: throw new IllegalArgumentException("Binary format type " + formatType + " is unknown"); } } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java index 8fa2537e1e2..5d1bc7b0c3f 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/DenseBinaryFormatTestCase.java @@ -55,7 +55,7 @@ public class DenseBinaryFormatTestCase { @Test public void requireThatFloatSerializationFormatDoNotChange() { - byte[] encodedTensor = new byte[]{4, // binary format type + byte[] encodedTensor = new byte[]{6, // binary format type 1, // float type 2, // dimension count 2, (byte) 'x', (byte) 'y', 2, // dimension xy with size |