diff options
Diffstat (limited to 'document/src/main')
3 files changed, 23 insertions, 10 deletions
diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index 104d63cae96..fd7ccfc5e96 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -54,7 +54,7 @@ public abstract class DataType extends Identifiable implements Serializable, Com public final static NumericDataType BYTE = new NumericDataType("byte", 16, ByteFieldValue.class, ByteFieldValue.getFactory()); public final static PrimitiveDataType PREDICATE = new PrimitiveDataType("predicate", 20, PredicateFieldValue.class, PredicateFieldValue.getFactory()); public final static int tensorDataTypeCode = 21; // All TensorDataType instances have id=21 but carries additional type information serialized separately - // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor + // ADDITIONAL parametrized types added at runtime: map, struct, array, weighted set, annotation reference, tensor // Tags are converted to weightedset<string> when reading the search definition TODO: Remove it public final static WeightedSetDataType TAG = new WeightedSetDataType(DataType.STRING, true, true); diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index 88353139b0f..9db80f3972b 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -29,7 +29,7 @@ public class TokenBuffer { } } - private Deque<Token> buffer; + private final Deque<Token> buffer; private int nesting = 0; public TokenBuffer() { diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index ad016a40fca..27426f584bd 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -11,6 +11,7 @@ import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; import static com.yahoo.document.json.readers.JsonParserHelpers.*; +import static com.yahoo.tensor.serialization.JsonFormat.decodeHexString; /** * Reads the tensor format defined at @@ -41,7 +42,7 @@ public class TensorReader { else if (TENSOR_BLOCKS.equals(buffer.currentName())) readTensorBlocks(buffer, builder); else if (builder.type().dimensions().stream().anyMatch(d -> d.isIndexed())) // sparse can be empty - throw new IllegalArgumentException("Expected a tensor value to contain either 'cells', 'values' or 'blocks'"); + throw new IllegalArgumentException("Expected a tensor value to contain either 'cells', 'values' or 'blocks', but got: "+buffer.currentName()); } expectObjectEnd(buffer.currentToken()); tensorFieldValue.assign(builder.build()); @@ -91,10 +92,18 @@ public class TensorReader { throw new IllegalArgumentException("The 'values' field can only be used with dense tensors. " + "Use 'cells' or 'blocks' instead"); IndexedTensor.BoundBuilder indexedBuilder = (IndexedTensor.BoundBuilder)builder; + if (buffer.currentToken() == JsonToken.VALUE_STRING) { + double[] decoded = decodeHexString(buffer.currentText(), builder.type().valueType()); + for (int i = 0; i < decoded.length; i++) { + indexedBuilder.cellByDirectIndex(i, decoded[i]); + } + return; + } int index = 0; int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { indexedBuilder.cellByDirectIndex(index++, readDouble(buffer)); + } expectCompositeEnd(buffer.currentToken()); } @@ -167,17 +176,21 @@ public class TensorReader { * @return the values read */ private static double[] readValues(TokenBuffer buffer, int size, TensorAddress address, TensorType type) { - expectArrayStart(buffer.currentToken()); - int index = 0; - int initNesting = buffer.nesting(); double[] values = new double[size]; - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) - values[index++] = readDouble(buffer); + if (buffer.currentToken() == JsonToken.VALUE_STRING) { + values = decodeHexString(buffer.currentText(), type.valueType()); + index = values.length; + } else { + expectArrayStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) + values[index++] = readDouble(buffer); + expectCompositeEnd(buffer.currentToken()); + } if (index != size) throw new IllegalArgumentException((address != null ? "At " + address.toString(type) + ": " : "") + "Expected " + size + " values, but got " + index); - expectCompositeEnd(buffer.currentToken()); return values; } |