diff options
| -rw-r--r-- | vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java | 37 | ||||
| -rw-r--r-- | vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java | 41 |
2 files changed, 78 insertions, 0 deletions
diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java index 32b36c5c5cb..67c81921803 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorParser.java @@ -5,6 +5,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import static com.yahoo.tensor.serialization.JsonFormat.decodeHexString; + /** * @author bratseth */ @@ -59,6 +61,9 @@ class TensorParser { return tensorFromDenseValueString(valueString, type, dimensionOrder); } else { + var t = maybeFromBinaryValueString(valueString, type, dimensionOrder); + if (t.isPresent()) { return t.get(); } + if (explicitType.isPresent() && ! explicitType.get().equals(TensorType.empty)) throw new IllegalArgumentException("Got a zero-dimensional tensor value ('" + tensorString + "') where type " + explicitType.get() + " is required"); @@ -118,6 +123,38 @@ class TensorParser { } } + private static Optional<Tensor> maybeFromBinaryValueString( + String valueString, + Optional<TensorType> type, + List<String> dimensionOrder) + { + if (type.isEmpty() || dimensionOrder != null) { + return Optional.empty(); + } + long sz = 1; + for (var d : type.get().dimensions()) { + sz *= d.size().orElse(0L); + } + if (sz == 0 || valueString.length() < sz * 2) { + return Optional.empty(); + } + try { + double[] values = decodeHexString(valueString, type.get().valueType()); + if (values.length != sz) { + return Optional.empty(); + } + var builder = IndexedTensor.Builder.of(type.get()); + var dib = (IndexedTensor.DirectIndexBuilder) builder; + for (int i = 0; i < sz; ++i) { + System.out.println("idx "+i+" -> "+values[i]); + dib.cellByDirectIndex(i, values[i]); + } + return Optional.of(builder.build()); + } catch (NumberFormatException e) { + return Optional.empty(); + } + } + private static Tensor tensorFromDenseValueString(String valueString, Optional<TensorType> type, List<String> dimensionOrder) { diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java index 53df95ec2e8..387cad49f5d 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorParserTestCase.java @@ -81,6 +81,47 @@ public class TensorParserTestCase { .cell( 5.0, 2, 0, 0) .cell(-6.0, 2, 1, 0).build(), Tensor.from("tensor( x[3],y[2],z[1]) : [1.0, 2.0, 3.0 , 4.0, 5, -6.0]")); + + var int8TT = TensorType.fromSpec("tensor<int8>(x[2],y[3])"); + + assertEquals("binary tensor A", + Tensor.Builder.of(int8TT) + .cell(1, 0, 0) + .cell(20, 0, 1) + .cell(127, 0, 2) + .cell(-1, 1, 0) + .cell(50, 1, 1) + .cell(-128, 1, 2).build(), + Tensor.from(int8TT, "01147FFF3280")); + + assertEquals("binary tensor B", + Tensor.Builder.of(int8TT) + .cell(26.0, 0, 0) + .cell(0.0, 0, 1) + .cell(31.0, 0, 2) + .cell(-68.0, 1, 0) + .cell(-98.0, 1, 1) + .cell(-34.0, 1, 2).build(), + Tensor.from(int8TT, "1a001fbc9ede")); + + assertEquals("binary tensor C", + Tensor.Builder.of(int8TT) + .cell(16, 0, 0) + .cell(32, 0, 1) + .cell(48, 0, 2) + .cell(-16, 1, 0) + .cell(-32, 1, 1) + .cell(-64, 1, 2).build(), + Tensor.from(int8TT, "102030F0E0C0")); + + var floatTT = TensorType.fromSpec("tensor<float>(x[3])"); + assertEquals("float tensor hexdump", + Tensor.Builder.of(floatTT) + .cell(0, 0) + .cell(1.25, 1) + .cell(-19.125, 2).build(), + Tensor.from(floatTT, "000000003FA00000c1990000")); + } @Test |