diff options
author | Geir Storli <geirst@verizonmedia.com> | 2019-01-30 14:55:55 +0100 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2019-01-30 15:01:06 +0100 |
commit | 047722083d075c8af5c710a912e753c37ace488f (patch) | |
tree | 340e6a263f93258d7bf58de5b794d66849c886ba /document | |
parent | 2f0ffe3f69791fed550a16c8f52f856d78762705 (diff) |
Add initial support in JSON parser for modify updates on tensor fields.
Diffstat (limited to 'document')
5 files changed, 214 insertions, 2 deletions
diff --git a/document/abi-spec.json b/document/abi-spec.json index 9ae12660abd..ddeeaeb46ec 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -5197,7 +5197,7 @@ "fields": [ "public static final enum com.yahoo.document.update.TensorModifyUpdate$Operation REPLACE", "public static final enum com.yahoo.document.update.TensorModifyUpdate$Operation ADD", - "public static final enum com.yahoo.document.update.TensorModifyUpdate$Operation MUL", + "public static final enum com.yahoo.document.update.TensorModifyUpdate$Operation MULTIPLY", "public final int id", "public final java.lang.String name" ] diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorModifyUpdateReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorModifyUpdateReader.java new file mode 100644 index 00000000000..c8483634a23 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorModifyUpdateReader.java @@ -0,0 +1,107 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.document.json.readers; + +import com.yahoo.document.Field; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.document.update.TensorModifyUpdate; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; +import static com.yahoo.document.json.readers.TensorReader.TENSOR_CELLS; +import static com.yahoo.document.json.readers.TensorReader.readTensorCells; + +/** + * Class used to read a modify update for a tensor field. + */ +public class TensorModifyUpdateReader { + + public static final String UPDATE_MODIFY = "modify"; + private static final String MODIFY_OPERATION = "operation"; + private static final String MODIFY_REPLACE = "replace"; + private static final String MODIFY_ADD = "add"; + private static final String MODIFY_MULTIPLY = "multiply"; + + public static TensorModifyUpdate createModifyUpdate(TokenBuffer buffer, Field field) { + + expectFieldIsOfTypeTensor(field); + expectObjectStart(buffer.currentToken()); + + ModifyUpdateResult result = createModifyUpdateResult(buffer, field); + expectOperationSpecified(result.operation, field.getName()); + expectTensorSpecified(result.tensor, field.getName()); + + return new TensorModifyUpdate(result.operation, result.tensor); + } + + private static void expectFieldIsOfTypeTensor(Field field) { + if (!(field.getDataType() instanceof TensorDataType)) { + throw new IllegalArgumentException("A modify update can only be applied to tensor fields. " + + "Field '" + field.getName() + "' is of type '" + field.getDataType().getName() + "'"); + } + } + + private static void expectOperationSpecified(TensorModifyUpdate.Operation operation, String fieldName) { + if (operation == null) { + throw new IllegalArgumentException("Modify update for field '" + fieldName + "' does not contain an operation"); + } + } + + private static void expectTensorSpecified(TensorFieldValue tensor, String fieldName) { + if (tensor == null) { + throw new IllegalArgumentException("Modify update for field '" + fieldName + "' does not contain tensor cells"); + } + } + + private static class ModifyUpdateResult { + TensorModifyUpdate.Operation operation = null; + TensorFieldValue tensor = null; + } + + private static ModifyUpdateResult createModifyUpdateResult(TokenBuffer buffer, Field field) { + ModifyUpdateResult result = new ModifyUpdateResult(); + // TODO: convert tensor type to one with only mapped dimensions. + TensorDataType tensorDataType = (TensorDataType)field.getDataType(); + buffer.next(); + int localNesting = buffer.nesting(); + while (localNesting <= buffer.nesting()) { + switch (buffer.currentName()) { + case MODIFY_OPERATION: + result.operation = createOperation(buffer, field.getName()); + break; + case TENSOR_CELLS: + result.tensor = createTensor(buffer, tensorDataType.getTensorType()); + break; + default: + throw new IllegalArgumentException("Unknown JSON string '" + buffer.currentName() + "' in modify update for field '" + field.getName() + "'"); + } + buffer.next(); + } + return result; + } + + private static TensorModifyUpdate.Operation createOperation(TokenBuffer buffer, String fieldName) { + switch (buffer.currentText()) { + case MODIFY_REPLACE: + return TensorModifyUpdate.Operation.REPLACE; + case MODIFY_ADD: + return TensorModifyUpdate.Operation.ADD; + case MODIFY_MULTIPLY: + return TensorModifyUpdate.Operation.MULTIPLY; + default: + throw new IllegalArgumentException("Unknown operation '" + buffer.currentText() + "' in modify update for field '" + fieldName + "'"); + } + } + + private static TensorFieldValue createTensor(TokenBuffer buffer, TensorType tensorType) { + Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorType); + readTensorCells(buffer, tensorBuilder); + TensorFieldValue result = new TensorFieldValue(tensorType); + result.assign(tensorBuilder.build()); + return result; + } + +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java index 6189c12c8c9..cbdade6ad3c 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/VespaJsonDocumentReader.java @@ -30,6 +30,8 @@ import static com.yahoo.document.json.readers.MapReader.UPDATE_MATCH; import static com.yahoo.document.json.readers.MapReader.createMapUpdate; import static com.yahoo.document.json.readers.SingleValueReader.UPDATE_ASSIGN; import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate; +import static com.yahoo.document.json.readers.TensorModifyUpdateReader.UPDATE_MODIFY; +import static com.yahoo.document.json.readers.TensorModifyUpdateReader.createModifyUpdate; /** * @author freva @@ -122,6 +124,9 @@ public class VespaJsonDocumentReader { case UPDATE_MATCH: fieldUpdate.addValueUpdate(createMapUpdate(buffer, field)); break; + case UPDATE_MODIFY: + fieldUpdate.addValueUpdate(createModifyUpdate(buffer, field)); + break; default: String action = buffer.currentName(); fieldUpdate.addValueUpdate(readSingleUpdate(buffer, field.getDataType(), action)); diff --git a/document/src/main/java/com/yahoo/document/update/TensorModifyUpdate.java b/document/src/main/java/com/yahoo/document/update/TensorModifyUpdate.java index 6f0c47e8c60..797a9d96a1a 100644 --- a/document/src/main/java/com/yahoo/document/update/TensorModifyUpdate.java +++ b/document/src/main/java/com/yahoo/document/update/TensorModifyUpdate.java @@ -80,7 +80,7 @@ public class TensorModifyUpdate extends ValueUpdate<TensorFieldValue> { /** * Multiply values from matching update tensor cells with target tensor cells. */ - MUL(2, "multiply"); + MULTIPLY(2, "multiply"); /** * The numeric ID of the operator, used for serialization. diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index b70ec0907e9..27cda8e73c2 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -43,6 +43,7 @@ import com.yahoo.document.update.AssignValueUpdate; import com.yahoo.document.update.ClearValueUpdate; import com.yahoo.document.update.FieldUpdate; import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.TensorModifyUpdate; import com.yahoo.document.update.ValueUpdate; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.IndexedTensor; @@ -1268,6 +1269,78 @@ public class JsonReaderTestCase { } @Test + public void tensor_modify_update_with_replace_operation() { + assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.REPLACE, "mappedtensorfield", + inputJson("{", + " 'operation': 'replace',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); + } + + @Test + public void tensor_modify_update_with_add_operation() { + assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.ADD, "mappedtensorfield", + inputJson("{", + " 'operation': 'add',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); + } + + @Test + public void tensor_modify_update_with_multiply_operation() { + assertTensorModifyUpdate("{{x:a,y:b}:2.0}", TensorModifyUpdate.Operation.MULTIPLY, "mappedtensorfield", + inputJson("{", + " 'operation': 'multiply',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}")); + } + + @Test + public void tensor_modify_update_on_non_tensor_field_throws() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("A modify update can only be applied to tensor fields. Field 'something' is of type 'string'"); + JsonReader reader = createReader(inputJson("{ 'update': 'id:unittest:smoke::doc1',", + " 'fields': {", + " 'something': {", + " 'modify': {} }}}")); + reader.readSingleDocument(DocumentParser.SupportedOperation.UPDATE, "id:unittest:smoke::doc1"); + } + + @Test + public void tensor_modify_update_with_unknown_operation_throws() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Unknown operation 'unknown' in modify update for field 'mappedtensorfield'"); + createTensorModifyUpdate(inputJson("{", + " 'operation': 'unknown',", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 } ]}"), "mappedtensorfield"); + } + + @Test + public void tensor_modify_update_without_operation_throws() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Modify update for field 'mappedtensorfield' does not contain an operation"); + createTensorModifyUpdate(inputJson("{", + " 'cells': [] }"), "mappedtensorfield"); + } + + @Test + public void tensor_modify_update_without_cells_throws() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Modify update for field 'mappedtensorfield' does not contain tensor cells"); + createTensorModifyUpdate(inputJson("{", + " 'operation': 'replace' }"), "mappedtensorfield"); + } + + @Test + public void tensor_modify_update_with_unknown_content_throws() { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Unknown JSON string 'unknown' in modify update for field 'mappedtensorfield'"); + createTensorModifyUpdate(inputJson("{", + " 'unknown': 'here' }"), "mappedtensorfield"); + } + + @Test public void require_that_parser_propagates_datatype_parser_errors_predicate() { assertParserErrorMatches( "Error in document 'id:unittest:testpredicate::0' - could not parse field 'boolean' of type 'predicate': " + @@ -1386,6 +1459,33 @@ public class JsonReaderTestCase { assertEquals(Tensor.from(expectedTensor), fieldValue.getTensor().get()); } + private DocumentUpdate createTensorModifyUpdate(String modifyJson, String tensorFieldName) { + JsonReader reader = createReader(inputJson("[", + "{ 'update': '" + TENSOR_DOC_ID + "',", + " 'fields': {", + " '" + tensorFieldName + "': {", + " 'modify': " + modifyJson + " }}}]")); + return (DocumentUpdate) reader.next(); + } + + private void assertTensorModifyUpdate(String expectedTensor, TensorModifyUpdate.Operation expectedOperation, + String tensorFieldName, String modifyJson) { + assertTensorModifyUpdate(expectedTensor, expectedOperation, tensorFieldName, + createTensorModifyUpdate(modifyJson, tensorFieldName)); + } + + private static void assertTensorModifyUpdate(String expectedTensor, TensorModifyUpdate.Operation expectedOperation, + String tensorFieldName, DocumentUpdate update) { + assertEquals("testtensor", update.getId().getDocType()); + assertEquals(TENSOR_DOC_ID, update.getId().toString()); + assertEquals(1, update.fieldUpdates().size()); + FieldUpdate fieldUpdate = update.getFieldUpdate(tensorFieldName); + assertEquals(1, fieldUpdate.size()); + TensorModifyUpdate modifyUpdate = (TensorModifyUpdate) fieldUpdate.getValueUpdate(0); + assertEquals(expectedOperation, modifyUpdate.getOperation()); + assertEquals(Tensor.from(expectedTensor), modifyUpdate.getValue().getTensor().get()); + } + private static FieldUpdate getTensorField(DocumentUpdate update) { FieldUpdate fieldUpdate = update.getFieldUpdate("mappedtensorfield"); assertEquals(1, fieldUpdate.size()); |