diff options
25 files changed, 278 insertions, 165 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index b70a88d09a0..a71082ecba3 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -899,19 +899,19 @@ ], "fields" : [ ] }, - "com.yahoo.prelude.query.MultiRangeItem$Limit": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ + "com.yahoo.prelude.query.MultiRangeItem$Limit" : { + "superClass" : "java.lang.Enum", + "interfaces" : [ ], + "attributes" : [ "public", "final", "enum" ], - "methods": [ + "methods" : [ "public static com.yahoo.prelude.query.MultiRangeItem$Limit[] values()", "public static com.yahoo.prelude.query.MultiRangeItem$Limit valueOf(java.lang.String)" ], - "fields": [ + "fields" : [ "public static final enum com.yahoo.prelude.query.MultiRangeItem$Limit INCLUSIVE", "public static final enum com.yahoo.prelude.query.MultiRangeItem$Limit EXCLUSIVE" ] @@ -5318,7 +5318,10 @@ "public void setSummaryFields(java.lang.String)", "public boolean getTensorShortForm()", "public void setTensorShortForm(java.lang.String)", + "public void setTensorFormat(java.lang.String)", "public void setTensorShortForm(boolean)", + "public boolean getTensorDirectValues()", + "public void setTensorDirectValues(boolean)", "public void prepare()", "public boolean equals(java.lang.Object)", "public int hashCode()" @@ -7678,6 +7681,7 @@ "public com.yahoo.data.access.Inspector inspect()", "public java.lang.String toJson()", "public java.lang.String toJson(boolean)", + "public java.lang.String toJson(boolean, boolean)", "public java.lang.StringBuilder writeJson(java.lang.StringBuilder)", "public java.lang.Double getDouble(java.lang.String)", "public com.yahoo.tensor.Tensor getTensor(java.lang.String)", @@ -8725,4 +8729,4 @@ ], "fields" : [ ] } -} +}
\ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index fd4519fdbb0..b949d1edabd 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -77,6 +77,9 @@ public class Presentation implements Cloneable { /** Whether to renders tensors in short form */ private boolean tensorShortForm = true; + /** Whether to renders tensors in short form */ + private boolean tensorDirectValues = false; // TODO: Flip default on Vespa 9 + /** Set of explicitly requested summary fields, instead of summary classes */ private Set<String> summaryFields = LazySet.newHashSet(); @@ -178,27 +181,41 @@ public class Presentation implements Cloneable { /** * Returns whether tensors should use short form in JSON and textual representations, see - * <a href="https://docs.vespa.ai/en/reference/document-json-format.html#tensor">https://docs.vespa.ai/en/reference/document-json-format.html#tensor</a> - * and <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form</a>. + * <a href="https://docs.vespa.ai/en/reference/document-json-format.html#tensor">https://docs.vespa.ai/en/reference/document-json-format.html#tensor</a>. * Default is true. */ public boolean getTensorShortForm() { return tensorShortForm; } + /** @deprecated use setTensorFormat(). */ + @Deprecated // TODO: Remove on Vespa 9 + public void setTensorShortForm(String value) { + setTensorFormat(value); + } /** * Sets whether tensors should use short form in JSON and textual representations from a string. * * @param value a string which must be either 'short' or 'long' * @throws IllegalArgumentException if any other value is passed */ - public void setTensorShortForm(String value) { - tensorShortForm = toTensorShortForm(value); - } - - private boolean toTensorShortForm(String value) { - return switch (value) { - case "short" -> true; - case "long" -> false; - default -> throw new IllegalArgumentException("Value must be 'long' or 'short', not '" + value + "'"); + public void setTensorFormat(String value) { + switch (value) { + case "short" : + tensorShortForm = true; + tensorDirectValues = false; + break; + case "long" : + tensorShortForm = false; + tensorDirectValues = false; + break; + case "short-value" : + tensorShortForm = true; + tensorDirectValues = true; + break; + case "long-value" : + tensorShortForm = false; + tensorDirectValues = true; + break; + default : throw new IllegalArgumentException("Value must be 'long', 'short', 'long-value', or 'short-value', not '" + value + "'"); }; } @@ -206,6 +223,19 @@ public class Presentation implements Cloneable { this.tensorShortForm = tensorShortForm; } + /** + * Returns whether tensor content should be rendered directly, or inside a JSON object containing a + * "type" entry having the tensor type, and a "cells"/"values"/"blocks" entry (depending on type), + * having the tensor content. See + * <a href="https://docs.vespa.ai/en/reference/document-json-format.html#tensor">https://docs.vespa.ai/en/reference/document-json-format.html#tensor</a>. + * Default is false: Render wrapped in a JSON object. + */ + public boolean getTensorDirectValues() { return tensorDirectValues; } + + public void setTensorDirectValues(boolean tensorDirectValues) { + this.tensorDirectValues = tensorDirectValues; + } + /** Prepares this for binary serialization. For internal use - see {@link Query#prepare} */ public void prepare() { if (highlight != null) diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index d5dc8120f29..e4a83972fae 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -303,7 +303,7 @@ public class QueryProperties extends Properties { } else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) { if (key.last().equals(Presentation.TENSORS)) - query.getPresentation().setTensorShortForm(asString(value, "short")); + query.getPresentation().setTensorFormat(asString(value, "short")); // TODO: Switch default to short-value on Vespa 9 else throwIllegalParameter(key.last(), Presentation.FORMAT); } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 9bb7e882a4b..9498f860f88 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -132,6 +132,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { volatile boolean jsonMapsAll = true; volatile boolean jsonWsetsAll = false; volatile boolean tensorShortForm = true; + volatile boolean tensorDirectValues = false; boolean convertDeep() { return (jsonDeepMaps || jsonWsets); } void init() { this.debugRendering = false; @@ -140,6 +141,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { this.jsonMapsAll = true; this.jsonWsetsAll = true; this.tensorShortForm = true; + this.tensorDirectValues = false; } void getSettings(Query q) { if (q == null) { @@ -154,7 +156,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { this.jsonMapsAll = props.getBoolean(WRAP_DEEP_MAPS, true); this.jsonWsetsAll = props.getBoolean(WRAP_WSETS, true); this.tensorShortForm = q.getPresentation().getTensorShortForm(); - } + this.tensorDirectValues = q.getPresentation().getTensorDirectValues(); + } } private volatile FieldConsumerSettings fieldConsumerSettings; @@ -776,7 +779,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } else if (field instanceof Tensor) { renderTensor(Optional.of((Tensor)field)); } else if (field instanceof FeatureData) { - generator().writeRawValue(((FeatureData)field).toJson(settings.tensorShortForm)); + generator().writeRawValue(((FeatureData)field).toJson(settings.tensorShortForm, settings.tensorDirectValues)); } else if (field instanceof Inspectable) { renderInspectorDirect(((Inspectable)field).inspect()); } else if (field instanceof JsonProducer) { @@ -821,11 +824,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generator().writeEndObject(); return; } - if (settings.tensorShortForm && 1==2) { - generator().writeRawValue(new String(JsonFormat.encodeShortForm(tensor.get()), StandardCharsets.UTF_8)); - } else { - generator().writeRawValue(new String(JsonFormat.encode(tensor.get()), StandardCharsets.UTF_8)); - } + generator().writeRawValue(new String(JsonFormat.encode(tensor.get(), settings.tensorShortForm, settings.tensorDirectValues), + StandardCharsets.UTF_8)); } private JsonGenerator generator() { diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 2cb5e0e07e9..421f19475a6 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -65,16 +65,20 @@ public class FeatureData implements Inspectable, JsonProducer { } public String toJson(boolean tensorShortForm) { + return toJson(tensorShortForm, false); + } + + public String toJson(boolean tensorShortForm, boolean tensorDirectValues) { if (this == empty) return "{}"; if (jsonForm != null) return jsonForm; - jsonForm = JsonRender.render(value, new Encoder(new StringBuilder(), true, tensorShortForm)).toString(); + jsonForm = JsonRender.render(value, new Encoder(new StringBuilder(), true, tensorShortForm, tensorDirectValues)).toString(); return jsonForm; } @Override public StringBuilder writeJson(StringBuilder target) { - return JsonRender.render(value, new Encoder(target, true, false)); + return JsonRender.render(value, new Encoder(target, true, false, false)); } /** @@ -173,17 +177,19 @@ public class FeatureData implements Inspectable, JsonProducer { private static class Encoder extends JsonRender.StringEncoder { private final boolean tensorShortForm; + private final boolean tensorDirectValues; - Encoder(StringBuilder out, boolean compact, boolean tensorShortForm) { + Encoder(StringBuilder out, boolean compact, boolean tensorShortForm, boolean tensorDirectValues) { super(out, compact); this.tensorShortForm = tensorShortForm; + this.tensorDirectValues = tensorDirectValues; } @Override public void encodeDATA(byte[] value) { // This could be done more efficiently ... Tensor tensor = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(value)); - byte[] encodedTensor = tensorShortForm ? JsonFormat.encodeShortForm(tensor) : JsonFormat.encodeWithType(tensor); + byte[] encodedTensor = JsonFormat.encode(tensor, tensorShortForm, tensorDirectValues); target().append(new String(encodedTensor, StandardCharsets.UTF_8)); } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 9486eeb92de..c1ede03a371 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -219,7 +219,7 @@ public class JsonRendererTestCase { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Could not set 'presentation.format.tensors' to 'unknown': Value must be 'long' or 'short', not 'unknown'", + assertEquals("Could not set 'presentation.format.tensors' to 'unknown': Value must be 'long', 'short', 'long-value', or 'short-value', not 'unknown'", Exceptions.toMessageString(e)); } } diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index a3cda4034ac..110564bea46 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -75,12 +75,12 @@ public class JsonSerializationHelper { } public static void serializeTensorField(JsonGenerator generator, FieldBase field, TensorFieldValue value, - boolean shortForm, boolean valueOnly) { + boolean shortForm, boolean directValues) { wrapIOException(() -> { fieldNameIfNotNull(generator, field); if (value.getTensor().isPresent()) { Tensor tensor = value.getTensor().get(); - byte[] encoded = shortForm ? JsonFormat.encodeShortForm(tensor) : JsonFormat.encodeWithType(tensor); + byte[] encoded = JsonFormat.encode(tensor, shortForm, directValues); generator.writeRawValue(new String(encoded, StandardCharsets.UTF_8)); } else { diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index 27a1dd150f3..33243ab832c 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -78,30 +78,20 @@ public class JsonWriter implements DocumentWriter { private final JsonGenerator generator; private final boolean tensorShortForm; + private final boolean tensorDirectValues; - // I really hate exception unsafe constructors, but the alternative - // requires generator to not be a final /** + * Creates a JsonWriter. * - * @param out - * the target output stream - * @throws RuntimeException - * if unable to create the internal JSON generator + * @param out the target output stream + * @throws RuntimeException if unable to create the internal JSON generator */ public JsonWriter(OutputStream out) { this(createPrivateGenerator(out)); } - public JsonWriter(OutputStream out, boolean tensorShortForm) { - this(createPrivateGenerator(out), tensorShortForm); - } - - private static JsonGenerator createPrivateGenerator(OutputStream out) { - try { - return jsonFactory.createGenerator(out); - } catch (IOException e) { - throw new RuntimeException(e); - } + public JsonWriter(OutputStream out, boolean tensorShortForm, boolean tensorDirectValues) { + this(createPrivateGenerator(out), tensorShortForm, tensorDirectValues); } /** @@ -110,16 +100,26 @@ public class JsonWriter implements DocumentWriter { * after having written a full Document instance. In other words, JsonWriter * will <i>not</i> take ownership of the generator. * - * @param generator - * the output JSON generator + * @param generator the output JSON generator + * @param tensorShortForm whether to use the short type-dependent form for tensor values + * @param tensorDirectValues whether to output tensor values directly or wrapped in a map also containing the type */ - public JsonWriter(JsonGenerator generator) { - this(generator, false); - } - - public JsonWriter(JsonGenerator generator, boolean tensorShortForm) { + public JsonWriter(JsonGenerator generator, boolean tensorShortForm, boolean tensorDirectValues) { this.generator = generator; this.tensorShortForm = tensorShortForm; + this.tensorDirectValues = tensorDirectValues; + } + + private static JsonGenerator createPrivateGenerator(OutputStream out) { + try { + return jsonFactory.createGenerator(out); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public JsonWriter(JsonGenerator generator) { + this(generator, false, false); } /** @@ -128,8 +128,7 @@ public class JsonWriter implements DocumentWriter { * updating this class. This implementation throws an exception if it is * reached. * - * @throws UnsupportedOperationException - * if invoked + * @throws UnsupportedOperationException if invoked */ @Override public void write(FieldBase field, FieldValue value) { @@ -217,7 +216,7 @@ public class JsonWriter implements DocumentWriter { @Override public void write(FieldBase field, TensorFieldValue value) { - serializeTensorField(generator, field, value, tensorShortForm, false); + serializeTensorField(generator, field, value, tensorShortForm, tensorDirectValues); } @Override @@ -265,12 +264,14 @@ public class JsonWriter implements DocumentWriter { * Utility method to easily serialize a single document. * * @param document the document to be serialized - * @param tensorShortForm whether tensors should be serialized in short form + * @param tensorShortForm whether tensors should be serialized in a type-dependent short form + * @param tensorDirectValues whether tensors should be serialized as direct values or wrapped in a + * map also containing the type * @return the input document serialised as UTF-8 encoded JSON */ - public static byte[] toByteArray(Document document, boolean tensorShortForm) { + public static byte[] toByteArray(Document document, boolean tensorShortForm, boolean tensorDirectValues) { ByteArrayOutputStream out = new ByteArrayOutputStream(); - JsonWriter writer = new JsonWriter(out, tensorShortForm); + JsonWriter writer = new JsonWriter(out, tensorShortForm, tensorDirectValues); writer.write(document); return out.toByteArray(); } @@ -282,8 +283,8 @@ public class JsonWriter implements DocumentWriter { * @return the input document serialised as UTF-8 encoded JSON */ public static byte[] toByteArray(Document document) { - // TODO Vespa 9: change tensorShortForm default to true - return toByteArray(document, false); + // TODO Vespa 9: change tensorShortForm and tensorDirectValues default to true + return toByteArray(document, false, false); } /** diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java index b9e67a65a8d..2d31a6b6734 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java @@ -6,18 +6,15 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentType; /** - * @author <a href="mailto:ravishar@yahoo-inc.com">ravishar</a> + * @author ravishar */ public interface DocumentWriter extends FieldWriter { - /** - * write out a document - * - * @param document - * document to be written - */ + + /** Writes a document. */ void write(Document document); void write(DocumentId id); void write(DocumentType type); + } diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index edf410b312e..eab33afc3e4 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -452,16 +452,16 @@ public class JsonWriterTestCase { doc.setFieldValue(tensorField, new TensorFieldValue(tensor)); assertEqualJson(asDocument(docId, "{ \"tensorfield\": {\"type\":\"tensor(x[3])\", \"cells\":[{\"address\":{\"x\":\"0\"},\"value\":1.0},{\"address\":{\"x\":\"1\"},\"value\":2.0},{\"address\":{\"x\":\"2\"},\"value\":3.0}]} }"), - writeDocument(doc, false)); + writeDocument(doc, false, false)); assertEqualJson(asDocument(docId, "{ \"tensorfield\": {\"type\":\"tensor(x[3])\", \"values\":[1.0, 2.0, 3.0] } }"), - writeDocument(doc, true)); + writeDocument(doc, true, false)); } - private byte[] writeDocument(Document doc, boolean tensorShortForm) throws IOException { + private byte[] writeDocument(Document doc, boolean tensorShortForm, boolean tensorDirectValues) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); JsonFactory factory = new JsonFactory(); JsonGenerator generator = factory.createGenerator(out); - JsonWriter writer = new JsonWriter(generator, tensorShortForm); + JsonWriter writer = new JsonWriter(generator, tensorShortForm, tensorDirectValues); writer.write(doc); return out.toByteArray(); } diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java index ef04b6641e5..1bcd6363d2d 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java +++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java @@ -91,15 +91,15 @@ public class ModelsEvaluationHandler extends ThreadedHttpRequestHandler { } } Tensor result = evaluator.evaluate(); - - Optional<String> format = property(request, "format.tensors"); - if (format.isPresent() && format.get().equalsIgnoreCase("long")) { - return new Response(200, JsonFormat.encode(result)); - } - else if (format.isPresent() && format.get().equalsIgnoreCase("string")) { - return new Response(200, result.toString().getBytes(StandardCharsets.UTF_8)); - } - return new Response(200, JsonFormat.encodeShortForm(result)); + return switch (property(request, "format.tensors").orElse("short").toLowerCase()) { + case "short" -> new Response(200, JsonFormat.encode(result, true, false)); + case "long" -> new Response(200, JsonFormat.encode(result, false, false)); + case "short-value" -> new Response(200, JsonFormat.encode(result, true, true)); + case "long-value" -> new Response(200, JsonFormat.encode(result, false, true)); + case "string" -> new Response(200, result.toString(true, true).getBytes(StandardCharsets.UTF_8)); + case "string-long " -> new Response(200, result.toString(true, false ).getBytes(StandardCharsets.UTF_8)); + default -> new ErrorResponse(400, "Unknown tensor format '" + property(request, "format.tensors") + "'"); + }; } private HttpResponse listAllModels(HttpRequest request) { diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java b/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java index 00531e373ee..5fabfca8737 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java @@ -25,7 +25,11 @@ class HandlerTester { return s -> true; } private static Predicate<String> matchString(String expected) { - return s -> expected.equals(s); + return s -> { + // System.out.println("Expected: " + expected); + // System.out.println("Actual: " + s); + return expected.equals(s); + }; } public static Predicate<String> matchJson(String... expectedJson) { var jExp = String.join("\n", expectedJson).replaceAll("'", "\""); diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index c0e5dd9ccda..50dbecaffce 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -107,7 +107,7 @@ public class ModelsEvaluationHandlerTest { properties.put("non-existing-binding", "-1"); properties.put("format.tensors", "long"); String url = "http://localhost/model-evaluation/v1/xgboost_2_2/eval"; - String expected = "{\"cells\":[{\"address\":{},\"value\":-7.936679999999999}]}"; + String expected = "{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":-7.936679999999999}]}"; handler.assertResponse(url, properties, 200, expected); } @@ -196,7 +196,7 @@ public class ModelsEvaluationHandlerTest { properties.put("Placeholder", inputTensorShortForm()); properties.put("format.tensors", "long"); String url = "http://localhost/model-evaluation/v1/mnist_softmax/default.add/eval"; - String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\",\"d1\":\"0\"},\"value\":-0.3546536862850189},{\"address\":{\"d0\":\"0\",\"d1\":\"1\"},\"value\":0.3759574592113495},{\"address\":{\"d0\":\"0\",\"d1\":\"2\"},\"value\":0.06054411828517914},{\"address\":{\"d0\":\"0\",\"d1\":\"3\"},\"value\":-0.251544713973999},{\"address\":{\"d0\":\"0\",\"d1\":\"4\"},\"value\":0.017951013520359993},{\"address\":{\"d0\":\"0\",\"d1\":\"5\"},\"value\":1.2899067401885986},{\"address\":{\"d0\":\"0\",\"d1\":\"6\"},\"value\":-0.10389615595340729},{\"address\":{\"d0\":\"0\",\"d1\":\"7\"},\"value\":0.6367976665496826},{\"address\":{\"d0\":\"0\",\"d1\":\"8\"},\"value\":-1.4136744737625122},{\"address\":{\"d0\":\"0\",\"d1\":\"9\"},\"value\":-0.2573896050453186}]}"; + String expected = "{\"type\":\"tensor(d0[],d1[10])\",\"cells\":[{\"address\":{\"d0\":\"0\",\"d1\":\"0\"},\"value\":-0.3546536862850189},{\"address\":{\"d0\":\"0\",\"d1\":\"1\"},\"value\":0.3759574592113495},{\"address\":{\"d0\":\"0\",\"d1\":\"2\"},\"value\":0.06054411828517914},{\"address\":{\"d0\":\"0\",\"d1\":\"3\"},\"value\":-0.251544713973999},{\"address\":{\"d0\":\"0\",\"d1\":\"4\"},\"value\":0.017951013520359993},{\"address\":{\"d0\":\"0\",\"d1\":\"5\"},\"value\":1.2899067401885986},{\"address\":{\"d0\":\"0\",\"d1\":\"6\"},\"value\":-0.10389615595340729},{\"address\":{\"d0\":\"0\",\"d1\":\"7\"},\"value\":0.6367976665496826},{\"address\":{\"d0\":\"0\",\"d1\":\"8\"},\"value\":-1.4136744737625122},{\"address\":{\"d0\":\"0\",\"d1\":\"9\"},\"value\":-0.2573896050453186}]}"; handler.assertResponse(url, properties, 200, expected); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java index 29795fbcd95..86f56e14e2d 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java @@ -83,7 +83,7 @@ public class OnnxEvaluationHandlerTest { properties.put("input2", "tensor<float>(d0[1]):[3]"); properties.put("format.tensors", "long"); String url = "http://localhost/model-evaluation/v1/add_mul/output1/eval"; - String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":6.0}]}"; // output1 is a mul + String expected = "{\"type\":\"tensor<float>(d0[1])\",\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":6.0}]}"; // output1 is a mul handler.assertResponse(url, properties, 200, expected); } @@ -94,7 +94,7 @@ public class OnnxEvaluationHandlerTest { properties.put("input2", "tensor<float>(d0[1]):[3]"); properties.put("format.tensors", "long"); String url = "http://localhost/model-evaluation/v1/add_mul/output2/eval"; - String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":5.0}]}"; // output2 is an add + String expected = "{\"type\":\"tensor<float>(d0[1])\",\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":5.0}]}"; // output2 is an add handler.assertResponse(url, properties, 200, expected); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 7a66ba2ee79..1e2ee3968fa 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -749,14 +749,25 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private boolean tensorShortForm() { if (request != null && request.parameters().containsKey("format.tensors") && - request.parameters().get("format.tensors").contains("long")) { + ( request.parameters().get("format.tensors").contains("long") + || request.parameters().get("format.tensors").contains("long-value"))) { return false; } return true; // default } + private boolean tensorDirectValues() { + if (request != null && + request.parameters().containsKey("format.tensors") && + ( request.parameters().get("format.tensors").contains("short-value") + || request.parameters().get("format.tensors").contains("long-value"))) { + return true; + } + return false; // TODO: Flip default on Vespa 9 + } + synchronized void writeSingleDocument(Document document) throws IOException { - new JsonWriter(json, tensorShortForm()).writeFields(document); + new JsonWriter(json, tensorShortForm(), tensorDirectValues()).writeFields(document); } synchronized void writeDocumentsArrayStart() throws IOException { @@ -775,7 +786,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { ByteArrayOutputStream myOut = new ByteArrayOutputStream(1); myOut.write(','); // Prepend rather than append, to avoid double memory copying. try (JsonGenerator myJson = jsonFactory.createGenerator(myOut)) { - new JsonWriter(myJson, tensorShortForm()).write(document); + new JsonWriter(myJson, tensorShortForm(), tensorDirectValues()).write(document); } docs.add(myOut); diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 973d0a98b24..cc6b8567b03 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -527,7 +527,7 @@ public class DocumentV1ApiTest { " \"id\": \"id:space:music::one\"," + " \"fields\": {" + " \"artist\": \"Tom Waits\"," + - " \"embedding\": { \"cells\": [{\"address\":{\"x\":\"0\"},\"value\":1.0},{\"address\":{\"x\":\"1\"},\"value\": 2.0},{\"address\":{\"x\":\"2\"},\"value\": 3.0}]}" + + " \"embedding\": { \"type\": \"tensor(x[3])\",\"cells\": [{\"address\":{\"x\":\"0\"},\"value\":1.0},{\"address\":{\"x\":\"1\"},\"value\": 2.0},{\"address\":{\"x\":\"2\"},\"value\": 3.0}]}" + " }" + "}", response.readAll()); assertEquals(200, response.getStatus()); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java index c40e2c21561..7f85f37436b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java @@ -308,6 +308,7 @@ public class SimpleFeeder implements ReplyHandler { return new DocumentUpdate(deserializer); } } + @Override public FeedOperation read() throws Exception { int read = readExact(in, prefix); @@ -352,8 +353,6 @@ public class SimpleFeeder implements ReplyHandler { return new JsonDestination(params.getDumpStream(), failure, numReplies); } - - @SuppressWarnings("deprecation") SimpleFeeder(FeederParams params) { inputStreams = params.getInputStreams(); out = params.getStdOut(); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java index 7e464431f9a..91837cb4b09 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/ClientParameters.java @@ -39,13 +39,15 @@ public class ClientParameters { public final boolean jsonOutput; // Output JSON tensors in short form public final boolean tensorShortForm; - + // Output JSON tensorvalues directly + public final boolean tensorDirectValues; private ClientParameters( boolean help, Iterator<String> documentIds, boolean printIdsOnly, String fieldSet, String route, String cluster, String configId, boolean showDocSize, double timeout, boolean noRetry, int traceLevel, - DocumentProtocol.Priority priority, boolean jsonOutput, boolean tensorShortForm) { + DocumentProtocol.Priority priority, boolean jsonOutput, boolean tensorShortForm, + boolean tensorDirectValues) { this.help = help; this.documentIds = documentIds; @@ -61,6 +63,7 @@ public class ClientParameters { this.priority = priority; this.jsonOutput = jsonOutput; this.tensorShortForm = tensorShortForm; + this.tensorDirectValues = tensorDirectValues; } public static class Builder { @@ -78,6 +81,7 @@ public class ClientParameters { private DocumentProtocol.Priority priority; private boolean jsonOutput; private boolean tensorShortForm; + private boolean tensorDirectValues; public Builder setHelp(boolean help) { this.help = help; @@ -149,10 +153,15 @@ public class ClientParameters { return this; } + public Builder setTensorDirectValues(boolean tensorDirectValues) { + this.tensorDirectValues = tensorDirectValues; + return this; + } + public ClientParameters build() { return new ClientParameters( help, documentIds, printIdsOnly, fieldSet, route, cluster, configId, - showDocSize, timeout, noRetry, traceLevel, priority, jsonOutput, tensorShortForm); + showDocSize, timeout, noRetry, traceLevel, priority, jsonOutput, tensorShortForm, tensorDirectValues); } } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java index b059ca6e62a..f13ed13b92a 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -39,6 +39,7 @@ public class CommandLineOptions { public static final String JSONOUTPUT_OPTION = "jsonoutput"; public static final String XMLOUTPUT_OPTION = "xmloutput"; public static final String SHORTTENSORS_OPTION = "shorttensors"; + public static final String DIRECTTENSORS_OPTION = "directtensors"; private final Options options = createOptions(); private final InputStream stdIn; @@ -167,6 +168,7 @@ public class CommandLineOptions { boolean jsonOutput = cl.hasOption(JSONOUTPUT_OPTION); boolean xmlOutput = cl.hasOption(XMLOUTPUT_OPTION); boolean shortTensors = cl.hasOption(SHORTTENSORS_OPTION); + boolean directTensors = cl.hasOption(DIRECTTENSORS_OPTION); int trace = getTrace(cl); DocumentProtocol.Priority priority = getPriority(cl); double timeout = getTimeout(cl); @@ -218,6 +220,7 @@ public class CommandLineOptions { .setTimeout(timeout) .setJsonOutput(!xmlOutput) .setTensorShortForm(shortTensors) + .setTensorDirectValues(directTensors) .build(); } catch (ParseException pe) { throw new IllegalArgumentException(pe.getMessage()); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java index 1a2f3424b3c..0f17fa587e4 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/DocumentRetriever.java @@ -168,7 +168,7 @@ public class DocumentRetriever { System.out.println(document.getId()); } else { if (params.jsonOutput) { - System.out.print(Utf8.toString(JsonWriter.toByteArray(document, params.tensorShortForm))); + System.out.print(Utf8.toString(JsonWriter.toByteArray(document, params.tensorShortForm, params.tensorDirectValues))); } else { System.out.print(document.toXML(" ")); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java index 2ac0510a2a3..0c7ad81f212 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -33,30 +33,33 @@ import java.util.logging.Logger; * @author Thomas Gundersen */ public class StdOutVisitorHandler extends VdsVisitHandler { + private static final Logger log = Logger.getLogger( StdOutVisitorHandler.class.getName()); - private boolean printIds; - private boolean indentXml; - private int processTimeMilliSecs; - private PrintStream out; + private final boolean printIds; + private final boolean indentXml; + private final int processTimeMilliSecs; + private final PrintStream out; private final boolean jsonOutput; private final boolean tensorShortForm; + private final boolean tensorDirectValues; - private VisitorDataHandler dataHandler; + private final VisitorDataHandler dataHandler; public StdOutVisitorHandler(boolean printIds, boolean indentXml, boolean showProgress, boolean showStatistics, boolean doStatistics, boolean abortOnClusterDown, int processtime, boolean jsonOutput, - boolean tensorShortForm) + boolean tensorShortForm, + boolean tensorDirectValues) { this(printIds, indentXml, showProgress, showStatistics, doStatistics, abortOnClusterDown, processtime, - jsonOutput, tensorShortForm, createStdOutPrintStream()); + jsonOutput, tensorShortForm, tensorDirectValues, createStdOutPrintStream()); } StdOutVisitorHandler(boolean printIds, boolean indentXml, boolean showProgress, boolean showStatistics, boolean doStatistics, boolean abortOnClusterDown, int processtime, boolean jsonOutput, - boolean tensorShortForm, PrintStream out) + boolean tensorShortForm, boolean tensorDirectValues, PrintStream out) { super(showProgress, showStatistics, abortOnClusterDown); this.printIds = printIds; @@ -64,6 +67,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { this.processTimeMilliSecs = processtime; this.jsonOutput = jsonOutput; this.tensorShortForm = tensorShortForm; + this.tensorDirectValues = tensorDirectValues; this.out = out; this.dataHandler = new DataHandler(doStatistics); } @@ -174,7 +178,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { private void writeJsonDocument(Document doc) throws IOException { writeFeedStartOrRecordSeparator(); - out.write(JsonWriter.toByteArray(doc, tensorShortForm)); + out.write(JsonWriter.toByteArray(doc, tensorShortForm, tensorDirectValues)); } @Override diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index f84cb7270bd..340d4a7eb81 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -371,6 +371,7 @@ public class VdsVisit { private int fullTimeout = 7 * 24 * 60 * 60 * 1000; private boolean jsonOutput = false; private boolean tensorShortForm = false; // TODO Vespa 9: change default to true + private boolean tensorDirectValues = false; // TODO Vespa 9: change default to true public VisitorParameters getVisitorParameters() { return visitorParameters; @@ -447,16 +448,25 @@ public class VdsVisit { public void setTensorShortForm(boolean tensorShortForm) { this.tensorShortForm = tensorShortForm; } + + public boolean tensorDirectValues() { + return tensorDirectValues; + } + + public void setTensorDirectValues(boolean tensorDirectValues) { + this.tensorDirectValues = tensorDirectValues; + } + } protected static class ArgumentParser { - private Options options; + private final Options options; public ArgumentParser(Options options) { this.options = options; } - public VdsVisitParameters parse(String args[]) throws org.apache.commons.cli.ParseException { + public VdsVisitParameters parse(String[] args) throws org.apache.commons.cli.ParseException { VdsVisitParameters allParams = new VdsVisitParameters(); VisitorParameters params = new VisitorParameters(""); CommandLineParser parser = new DefaultParser(); @@ -572,6 +582,9 @@ public class VdsVisit { if (line.hasOption("shorttensors")) { allParams.setTensorShortForm(true); } + if (line.hasOption("tensorvalues")) { + allParams.setTensorDirectValues(true); + } boolean jsonOutput = line.hasOption("jsonoutput"); boolean xmlOutput = line.hasOption("xmloutput"); @@ -743,7 +756,8 @@ public class VdsVisit { params.getAbortOnClusterDown(), params.getProcessTime(), params.jsonOutput, - params.tensorShortForm); + params.tensorShortForm, + params.tensorDirectValues); if (visitorParameters.getResumeFileName() != null) { handler.setProgressFileName(visitorParameters.getResumeFileName()); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java index a2e9f91d503..c1bbe8711a5 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java @@ -42,7 +42,7 @@ public class StdOutVisitorHandlerTest { initStdOutVisitorHandlerTest(jsonOutput); ByteArrayOutputStream out = new ByteArrayOutputStream(); StdOutVisitorHandler visitorHandler = - new StdOutVisitorHandler(/*printIds*/true, false, false, false, false, false, 0, jsonOutput, false, new PrintStream(out, true)); + new StdOutVisitorHandler(/*printIds*/true, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true)); VisitorDataHandler dataHandler = visitorHandler.getDataHandler(); dataHandler.onDone(); String output = out.toString(); @@ -55,7 +55,7 @@ public class StdOutVisitorHandlerTest { initStdOutVisitorHandlerTest(jsonOutput); ByteArrayOutputStream out = new ByteArrayOutputStream(); StdOutVisitorHandler visitorHandler = - new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, 0, jsonOutput, false, new PrintStream(out, true)); + new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true)); VisitorDataHandler dataHandler = visitorHandler.getDataHandler(); dataHandler.onDone(); String expectedOutput = jsonOutput ? "[]" : ""; @@ -63,7 +63,7 @@ public class StdOutVisitorHandlerTest { assertEquals(expectedOutput, output); } - void do_test_json_tensor_fields_can_be_output_in_short_or_long_form(boolean tensorShortForm, String expectedOutput) { + void do_test_json_tensor_fields_rendering(boolean tensorShortForm, boolean tensorDirectValues, String expectedOutput) { var docType = new DocumentType("foo"); docType.addField("bar", TensorDataType.getTensor(TensorType.fromSpec("tensor(x[3])"))); var doc = new Document(docType, "id:baz:foo::tensor-stuff"); @@ -72,7 +72,7 @@ public class StdOutVisitorHandlerTest { var out = new ByteArrayOutputStream(); var visitorHandler = new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, - 0, true, tensorShortForm, new PrintStream(out, true)); + 0, true, tensorShortForm, tensorDirectValues, new PrintStream(out, true)); var dataHandler = visitorHandler.getDataHandler(); var controlSession = mock(VisitorControlSession.class); var ackToken = mock(AckToken.class); @@ -88,8 +88,8 @@ public class StdOutVisitorHandlerTest { void json_tensor_fields_can_be_output_in_long_form() { var expectedOutput = """ [ - {"id":"id:baz:foo::tensor-stuff","fields":{"bar":{"cells":[{"address":{"x":"0"},"value":1.0},{"address":{"x":"1"},"value":2.0},{"address":{"x":"2"},"value":3.0}]}}}]"""; - do_test_json_tensor_fields_can_be_output_in_short_or_long_form(false, expectedOutput); + {"id":"id:baz:foo::tensor-stuff","fields":{"bar":{"type":"tensor(x[3])","cells":[{"address":{"x":"0"},"value":1.0},{"address":{"x":"1"},"value":2.0},{"address":{"x":"2"},"value":3.0}]}}}]"""; + do_test_json_tensor_fields_rendering(false, false, expectedOutput); } @Test @@ -97,7 +97,7 @@ public class StdOutVisitorHandlerTest { var expectedOutput = """ [ {"id":"id:baz:foo::tensor-stuff","fields":{"bar":{"type":"tensor(x[3])","values":[1.0,2.0,3.0]}}}]"""; - do_test_json_tensor_fields_can_be_output_in_short_or_long_form(true, expectedOutput); + do_test_json_tensor_fields_rendering(true, false, expectedOutput); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index 8322b3b6327..68997c82d3e 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -37,52 +37,79 @@ import java.util.stream.Collectors; */ public class JsonFormat { - /** Serializes the given tensor value into JSON format */ - public static byte[] encode(Tensor tensor) { + /** + * Serializes the given tensor value into JSON format. + * + * @param tensor the tensor to serialize + * @param shortForm whether to encode in a short type-dependent format + * @param directValues whether to encode values directly, or wrapped in am object containing "type" and "cells" + */ + public static byte[] encode(Tensor tensor, boolean shortForm, boolean directValues) { Slime slime = new Slime(); - Cursor root = slime.setObject(); - encodeCells(tensor, root); + if (shortForm) { + Cursor root = null; + if ( ! directValues) { + root = slime.setObject(); + root.setString("type", tensor.type().toString()); + } + + if (tensor instanceof IndexedTensor denseTensor) { + // Encode as nested lists if indexed tensor + Cursor parent = root == null ? slime.setArray() : root.setArray("values"); + encodeValues(denseTensor, parent, new long[denseTensor.dimensionSizes().dimensions()], 0); + } else if (tensor instanceof MappedTensor && tensor.type().dimensions().size() == 1) { + // Short form for a single mapped dimension + Cursor parent = root == null ? slime.setObject() : root.setObject("cells"); + encodeSingleDimensionCells((MappedTensor) tensor, parent); + } else if (tensor instanceof MixedTensor && + tensor.type().dimensions().stream().anyMatch(TensorType.Dimension::isMapped)) { + // Short form for a mixed tensor + boolean singleMapped = tensor.type().dimensions().stream().filter(TensorType.Dimension::isMapped).count() == 1; + Cursor parent = root == null ? ( singleMapped ? slime.setObject() : slime.setArray() ) + : ( singleMapped ? root.setObject("blocks") : root.setArray("blocks")); + encodeBlocks((MixedTensor) tensor, parent); + } else { + // default to standard cell address output + Cursor parent = root == null ? slime.setArray() : root.setArray("cells"); + encodeCells(tensor, parent); + } + + return com.yahoo.slime.JsonFormat.toJsonBytes(slime); + } + else { + Cursor root = slime.setObject(); + root.setString("type", tensor.type().toString()); + encodeCells(tensor, root.setArray("cells")); + } return com.yahoo.slime.JsonFormat.toJsonBytes(slime); } - /** Serializes the given tensor type and value into JSON format */ + /** Serializes the given tensor value into JSON format, in long format, wrapped in an object containing "cells" only. */ + public static byte[] encode(Tensor tensor) { + return encode(tensor, false, false); + } + + /** + * Serializes the given tensor type and value into JSON format. + * + * @deprecated use #encode(#Tensor, boolean, boolean) + */ + @Deprecated // TODO: Remove on Vespa 9 public static byte[] encodeWithType(Tensor tensor) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("type", tensor.type().toString()); - encodeCells(tensor, root); - return com.yahoo.slime.JsonFormat.toJsonBytes(slime); + return encode(tensor, false, false); } - /** Serializes the given tensor type and value into a short-form JSON format */ + /** + * Serializes the given tensor type and value into a short-form JSON format. + * + * @deprecated use #encode(#Tensor, boolean, boolean) + */ + @Deprecated // TODO: Remove on Vespa 9 public static byte[] encodeShortForm(Tensor tensor) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - root.setString("type", tensor.type().toString()); - - if (tensor instanceof IndexedTensor denseTensor) { - // Encode as nested lists if indexed tensor - encodeValues(denseTensor, root.setArray("values"), new long[denseTensor.dimensionSizes().dimensions()], 0); - } - else if (tensor instanceof MappedTensor && tensor.type().dimensions().size() == 1) { - // Short form for a single mapped dimension - encodeSingleDimensionCells((MappedTensor) tensor, root); - } - else if (tensor instanceof MixedTensor && - tensor.type().dimensions().stream().filter(TensorType.Dimension::isMapped).count() >= 1) { - // Short form for a mixed tensor - encodeBlocks((MixedTensor) tensor, root); - } - else { - // default to standard cell address output - encodeCells(tensor, root); - } - - return com.yahoo.slime.JsonFormat.toJsonBytes(slime); + return encode(tensor, true, false); } - private static void encodeCells(Tensor tensor, Cursor rootObject) { - Cursor cellsArray = rootObject.setArray("cells"); + private static void encodeCells(Tensor tensor, Cursor cellsArray) { for (Iterator<Tensor.Cell> i = tensor.cellIterator(); i.hasNext(); ) { Tensor.Cell cell = i.next(); Cursor cellObject = cellsArray.addObject(); @@ -91,8 +118,7 @@ public class JsonFormat { } } - private static void encodeSingleDimensionCells(MappedTensor tensor, Cursor cursor) { - Cursor cells = cursor.setObject("cells"); + private static void encodeSingleDimensionCells(MappedTensor tensor, Cursor cells) { if (tensor.type().dimensions().size() > 1) throw new IllegalStateException("JSON encode of mapped tensor can only contain a single dimension"); tensor.cells().forEach((k,v) -> cells.setDouble(k.label(0), v)); @@ -124,7 +150,6 @@ public class JsonFormat { if (mappedDimensions.size() < 1) { throw new IllegalArgumentException("Should be ensured by caller"); } - cursor = (mappedDimensions.size() == 1) ? cursor.setObject("blocks") : cursor.setArray("blocks"); // Create tensor type for mapped dimensions subtype TensorType mappedSubType = new TensorType.Builder(mappedDimensions).build(); diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java index 1cbf1709be1..4692cf87d59 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/JsonFormatTestCase.java @@ -47,8 +47,8 @@ public class JsonFormatTestCase { builder.cell().label("x", "a").label("y", "b").value(2.0); builder.cell().label("x", "c").label("y", "d").value(3.0); Tensor tensor = builder.build(); - byte[] json = JsonFormat.encode(tensor); - assertEquals("{\"cells\":[" + + byte[] json = JsonFormat.encode(tensor, false, false); + assertEquals("{\"type\":\"tensor(x{},y{})\",\"cells\":[" + "{\"address\":{\"x\":\"a\",\"y\":\"b\"},\"value\":2.0}," + "{\"address\":{\"x\":\"c\",\"y\":\"d\"},\"value\":3.0}" + "]}", @@ -61,8 +61,8 @@ public class JsonFormatTestCase { public void testEmptySparseTensor() { Tensor.Builder builder = Tensor.Builder.of(TensorType.fromSpec("tensor(x{},y{})")); Tensor tensor = builder.build(); - byte[] json = JsonFormat.encode(tensor); - assertEquals("{\"cells\":[]}", + byte[] json = JsonFormat.encode(tensor, false, false); + assertEquals("{\"type\":\"tensor(x{},y{})\",\"cells\":[]}", new String(json, StandardCharsets.UTF_8)); Tensor decoded = JsonFormat.decode(tensor.type(), json); assertEquals(tensor, decoded); @@ -95,8 +95,8 @@ public class JsonFormatTestCase { builder.cell().label("x", 1).label("y", 0).value(5.0); builder.cell().label("x", 1).label("y", 1).value(7.0); Tensor tensor = builder.build(); - byte[] json = JsonFormat.encode(tensor); - assertEquals("{\"cells\":[" + + byte[] json = JsonFormat.encode(tensor, false, false); + assertEquals("{\"type\":\"tensor(x[2],y[2])\",\"cells\":[" + "{\"address\":{\"x\":\"0\",\"y\":\"0\"},\"value\":2.0}," + "{\"address\":{\"x\":\"0\",\"y\":\"1\"},\"value\":3.0}," + "{\"address\":{\"x\":\"1\",\"y\":\"0\"},\"value\":5.0}," + @@ -206,9 +206,14 @@ public class JsonFormatTestCase { builder.cell().label("x", 1).label("y", 1).value(0.0); builder.cell().label("x", 1).label("y", 2).value(42.0); Tensor expected = builder.build(); + String denseJson = "{\"values\":\"027FFF80002A\"}"; Tensor decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8)); assertEquals(expected, decoded); + + denseJson = "\"027FFF80002A\""; + decoded = JsonFormat.decode(expected.type(), denseJson.getBytes(StandardCharsets.UTF_8)); + assertEquals(expected, decoded); } @Test @@ -233,6 +238,7 @@ public class JsonFormatTestCase { builder.cell().label("x", 1).label("y", 1).value(6.0); builder.cell().label("x", 1).label("y", 2).value(7.0); Tensor expected = builder.build(); + String mixedJson = "{\"blocks\":[" + "{\"address\":{\"x\":\"0\"},\"values\":\"020304\"}," + "{\"address\":{\"x\":\"1\"},\"values\":\"050607\"}" + @@ -375,7 +381,7 @@ public class JsonFormatTestCase { } private void assertEncodeDecode(Tensor tensor) { - Tensor decoded = JsonFormat.decode(tensor.type(), JsonFormat.encodeWithType(tensor)); + Tensor decoded = JsonFormat.decode(tensor.type(), JsonFormat.encode(tensor, false, false)); assertEquals(tensor, decoded); assertEquals(tensor.type(), decoded.type()); } @@ -403,7 +409,7 @@ public class JsonFormatTestCase { } private void assertEncodeShortForm(Tensor tensor, String expected) { - byte[] json = JsonFormat.encodeShortForm(tensor); + byte[] json = JsonFormat.encode(tensor, true, false); assertEquals(expected, new String(json, StandardCharsets.UTF_8)); } |