diff options
Diffstat (limited to 'document/src/main')
16 files changed, 785 insertions, 596 deletions
diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 42cc5c5b065..8c48be01799 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -6,10 +6,6 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.google.common.annotations.Beta; import com.google.common.base.Preconditions; -import com.yahoo.collections.Pair; -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; @@ -19,33 +15,26 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; -import com.yahoo.document.MapDataType; -import com.yahoo.document.PositionDataType; -import com.yahoo.document.ReferenceDataType; import com.yahoo.document.TestAndSetCondition; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.document.datatypes.CollectionFieldValue; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.MapFieldValue; -import com.yahoo.document.datatypes.StructuredFieldValue; -import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.document.datatypes.WeightedSet; -import com.yahoo.document.json.TokenBuffer.Token; +import com.yahoo.document.json.document.DocumentParser; +import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.update.FieldUpdate; -import com.yahoo.document.update.MapValueUpdate; -import com.yahoo.document.update.ValueUpdate; -import com.yahoo.tensor.MappedTensor; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import org.apache.commons.codec.binary.Base64; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; +import static com.yahoo.document.json.document.DocumentParser.parseDocumentsFields; +import static com.yahoo.document.json.readers.AddRemoveCreator.createAdds; +import static com.yahoo.document.json.readers.AddRemoveCreator.createRemoves; +import static com.yahoo.document.json.readers.CompositeReader.populateComposite; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; +import static com.yahoo.document.json.readers.MapReader.UPDATE_MATCH; +import static com.yahoo.document.json.readers.MapReader.createMapUpdate; +import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate; + /** * Initialize Vespa documents/updates/removes from an InputStream containing a * valid JSON representation of a feed. @@ -56,50 +45,18 @@ import java.util.Optional; @Beta public class JsonReader { - private enum FieldOperation { - ADD, REMOVE + // Only used for testing. + public Optional<DocumentParseInfo> parseDocument() { + return DocumentParser.parseDocument(parser); } - static final String MAP_KEY = "key"; - static final String MAP_VALUE = "value"; - static final String FIELDS = "fields"; - static final String REMOVE = "remove"; - static final String UPDATE_INCREMENT = "increment"; - static final String UPDATE_DECREMENT = "decrement"; - static final String UPDATE_MULTIPLY = "multiply"; - static final String UPDATE_DIVIDE = "divide"; - static final String TENSOR_DIMENSIONS = "dimensions"; - static final String TENSOR_CELLS = "cells"; - static final String TENSOR_ADDRESS = "address"; - static final String TENSOR_VALUE = "value"; - - private static final String UPDATE = "update"; - private static final String PUT = "put"; - private static final String ID = "id"; - private static final String CONDITION = "condition"; - private static final String CREATE_IF_NON_EXISTENT = "create"; - private static final String UPDATE_ASSIGN = "assign"; private static final String UPDATE_REMOVE = "remove"; - private static final String UPDATE_MATCH = "match"; private static final String UPDATE_ADD = "add"; - private static final String UPDATE_ELEMENT = "element"; private final JsonParser parser; - private TokenBuffer buffer = new TokenBuffer(); private final DocumentTypeManager typeManager; private ReaderState state = ReaderState.AT_START; - static class DocumentParseInfo { - public DocumentId documentId; - public Optional<Boolean> create = Optional.empty(); - Optional<String> condition = Optional.empty(); - SupportedOperation operationType = null; - } - - enum SupportedOperation { - PUT, UPDATE, REMOVE - } - enum ReaderState { AT_START, READING, END_OF_FEED } @@ -121,11 +78,11 @@ public class JsonReader { * @param docIdString document ID. * @return the document */ - public DocumentOperation readSingleDocument(SupportedOperation operationType, String docIdString) { + public DocumentOperation readSingleDocument(DocumentParser.SupportedOperation operationType, String docIdString) { DocumentId docId = new DocumentId(docIdString); - DocumentParseInfo documentParseInfo = parseToDocumentsFieldsAndInsertFieldsIntoBuffer(docId); + DocumentParseInfo documentParseInfo = parseDocumentsFields(parser, docId); documentParseInfo.operationType = operationType; - DocumentOperation operation = createDocumentOperation(documentParseInfo); + DocumentOperation operation = createDocumentOperation(documentParseInfo.fieldsBuffer, documentParseInfo); operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition)); return operation; } @@ -133,7 +90,7 @@ public class JsonReader { public DocumentOperation next() { switch (state) { case AT_START: - JsonToken t = nextToken(); + JsonToken t = nextToken(parser); expectArrayStart(t); state = ReaderState.READING; break; @@ -143,34 +100,34 @@ public class JsonReader { break; } - Optional<DocumentParseInfo> documentParseInfo = parseDocument(); + Optional<DocumentParseInfo> documentParseInfo = DocumentParser.parseDocument(parser); if (! documentParseInfo.isPresent()) { state = ReaderState.END_OF_FEED; return null; } - DocumentOperation operation = createDocumentOperation(documentParseInfo.get()); + DocumentOperation operation = createDocumentOperation(documentParseInfo.get().fieldsBuffer, documentParseInfo.get()); operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.get().condition)); return operation; } - private DocumentOperation createDocumentOperation(DocumentParseInfo documentParseInfo) { + private DocumentOperation createDocumentOperation(TokenBuffer buffer, DocumentParseInfo documentParseInfo) { DocumentType documentType = getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager); final DocumentOperation documentOperation; try { switch (documentParseInfo.operationType) { case PUT: documentOperation = new DocumentPut(new Document(documentType, documentParseInfo.documentId)); - readPut((DocumentPut) documentOperation); - verifyEndState(); + readPut(buffer, (DocumentPut) documentOperation); + verifyEndState(buffer); break; case REMOVE: documentOperation = new DocumentRemove(documentParseInfo.documentId); break; case UPDATE: documentOperation = new DocumentUpdate(documentType, documentParseInfo.documentId); - readUpdate((DocumentUpdate) documentOperation); - verifyEndState(); + readUpdate(buffer, (DocumentUpdate) documentOperation); + verifyEndState(buffer); break; default: throw new IllegalStateException("Implementation out of sync with itself. This is a bug."); @@ -188,83 +145,34 @@ public class JsonReader { return documentOperation; } - void readUpdate(DocumentUpdate next) { + // Exposed for unit testing... + void readUpdate(TokenBuffer buffer, DocumentUpdate next) { if (buffer.size() == 0) { - bufferFields(nextToken()); + buffer.bufferObject(nextToken(parser), parser); } - populateUpdateFromBuffer(next); + populateUpdateFromBuffer(buffer, next); } - void readPut(DocumentPut put) { + // Exposed for unit testing... + void readPut(TokenBuffer buffer, DocumentPut put) { if (buffer.size() == 0) { - bufferFields(nextToken()); + buffer.bufferObject(nextToken(parser), parser); } - JsonToken t = buffer.currentToken(); try { - populateComposite(put.getDocument(), t); + populateComposite(buffer, put.getDocument()); } catch (JsonReaderException e) { throw JsonReaderException.addDocId(e, put.getId()); } } - private DocumentParseInfo parseToDocumentsFieldsAndInsertFieldsIntoBuffer(DocumentId documentId) { - long indentLevel = 0; - DocumentParseInfo documentParseInfo = new DocumentParseInfo(); - documentParseInfo.documentId = documentId; - while (true) { - // we should now be at the start of a feed operation or at the end of the feed - JsonToken t = nextToken(); - if (t == null) { - throw new IllegalArgumentException("Could not read document, no document?"); - } - switch (t) { - case START_OBJECT: - indentLevel++; - break; - case END_OBJECT: - indentLevel--; - break; - case START_ARRAY: - indentLevel+=10000L; - break; - case END_ARRAY: - indentLevel-=10000L; - break; - } - if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) { - try { - if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { - documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue()); - continue; - } - } catch (IOException e) { - throw new RuntimeException("Got IO exception while parsing document", e); - } - } - if (indentLevel == 2L && t == JsonToken.START_OBJECT) { - - try { - if (!FIELDS.equals(parser.getCurrentName())) { - continue; - } - } catch (IOException e) { - throw new RuntimeException("Got IO exception while parsing document", e); - } - bufferFields(t); - break; - } - } - return documentParseInfo; - } - - private void verifyEndState() { + private void verifyEndState(TokenBuffer buffer) { Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation"); expectObjectEnd(buffer.currentToken()); Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation"); Preconditions.checkState(buffer.size() == 0, "Dangling data at end of operation"); } - private void populateUpdateFromBuffer(DocumentUpdate update) { + private static void populateUpdateFromBuffer(TokenBuffer buffer, DocumentUpdate update) { expectObjectStart(buffer.currentToken()); int localNesting = buffer.nesting(); JsonToken t = buffer.next(); @@ -273,12 +181,12 @@ public class JsonReader { expectObjectStart(t); String fieldName = buffer.currentName(); Field field = update.getType().getField(fieldName); - addFieldUpdates(update, field); + addFieldUpdates(buffer, update, field); t = buffer.next(); } } - private void addFieldUpdates(DocumentUpdate update, Field field) { + private static void addFieldUpdates(TokenBuffer buffer, DocumentUpdate update, Field field) { int localNesting = buffer.nesting(); FieldUpdate fieldUpdate = FieldUpdate.create(field); @@ -286,457 +194,24 @@ public class JsonReader { while (localNesting <= buffer.nesting()) { switch (buffer.currentName()) { case UPDATE_REMOVE: - createAddsOrRemoves(field, fieldUpdate, FieldOperation.REMOVE); + createRemoves(buffer, field, fieldUpdate); break; case UPDATE_ADD: - createAddsOrRemoves(field, fieldUpdate, FieldOperation.ADD); + createAdds(buffer, field, fieldUpdate); break; case UPDATE_MATCH: - fieldUpdate.addValueUpdate(createMapUpdate(field)); + fieldUpdate.addValueUpdate(createMapUpdate(buffer, field)); break; default: String action = buffer.currentName(); - fieldUpdate.addValueUpdate(readSingleUpdate(field.getDataType(), action)); + fieldUpdate.addValueUpdate(readSingleUpdate(buffer, field.getDataType(), action)); } buffer.next(); } update.addFieldUpdate(fieldUpdate); } - @SuppressWarnings("rawtypes") - private ValueUpdate createMapUpdate(Field field) { - buffer.next(); - MapValueUpdate m = (MapValueUpdate) createMapUpdate(field.getDataType(), null, null); - buffer.next(); - // must generate the field value in parallell with the actual - return m; - - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private ValueUpdate createMapUpdate(DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) { - TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT); - if (UPDATE_ELEMENT.equals(buffer.currentName())) { - buffer.next(); - } - - FieldValue key = keyTypeForMapUpdate(element, currentLevel); - if (keyParent != null) { - ((CollectionFieldValue) keyParent).add(key); - } - // structure is: [(match + element)*, (element + action)] - // match will always have element, and either match or action - if (!UPDATE_MATCH.equals(buffer.currentName())) { - // we have reached an action... - if (topLevelKey == null) { - return ValueUpdate.createMap(key, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName())); - } else { - return ValueUpdate.createMap(topLevelKey, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName())); - } - } else { - // next level of matching - if (topLevelKey == null) { - return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, key); - } else { - return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, topLevelKey); - } - } - } - - private DataType valueTypeForMapUpdate(DataType parentType) { - if (parentType instanceof WeightedSetDataType) { - return DataType.INT; - } else if (parentType instanceof CollectionDataType) { - return ((CollectionDataType) parentType).getNestedType(); - } else if (parentType instanceof MapDataType) { - return ((MapDataType) parentType).getValueType(); - } else { - throw new UnsupportedOperationException("Unexpected parent type: " + parentType); - } - } - - private FieldValue keyTypeForMapUpdate(Token element, DataType expectedType) { - FieldValue v; - if (expectedType instanceof ArrayDataType) { - v = new IntegerFieldValue(Integer.valueOf(element.text)); - } else if (expectedType instanceof WeightedSetDataType) { - v = ((WeightedSetDataType) expectedType).getNestedType().createFieldValue(element.text); - } else if (expectedType instanceof MapDataType) { - v = ((MapDataType) expectedType).getKeyType().createFieldValue(element.text); - } else { - throw new IllegalArgumentException("Container type " + expectedType + " not supported for match update."); - } - return v; - } - - @SuppressWarnings("rawtypes") - private ValueUpdate readSingleUpdate(DataType expectedType, String action) { - ValueUpdate update; - - switch (action) { - case UPDATE_ASSIGN: - update = (buffer.currentToken() == JsonToken.VALUE_NULL) - ? ValueUpdate.createClear() - : ValueUpdate.createAssign(readSingleValue(buffer.currentToken(), expectedType)); - break; - // double is silly, but it's what is used internally anyway - case UPDATE_INCREMENT: - update = ValueUpdate.createIncrement(Double.valueOf(buffer.currentText())); - break; - case UPDATE_DECREMENT: - update = ValueUpdate.createDecrement(Double.valueOf(buffer.currentText())); - break; - case UPDATE_MULTIPLY: - update = ValueUpdate.createMultiply(Double.valueOf(buffer.currentText())); - break; - case UPDATE_DIVIDE: - update = ValueUpdate.createDivide(Double.valueOf(buffer.currentText())); - break; - default: - throw new IllegalArgumentException("Operation \"" + buffer.currentName() + "\" not implemented."); - } - return update; - } - - // yes, this suppresswarnings ugliness is by intention, the code relies on - // the contracts in the builders - @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) - private void createAddsOrRemoves(Field field, FieldUpdate update, FieldOperation op) { - FieldValue container = field.getDataType().createFieldValue(); - FieldUpdate singleUpdate; - int initNesting = buffer.nesting(); - JsonToken token; - - Preconditions.checkState(buffer.currentToken().isStructStart(), "Expected start of composite, got %s", buffer.currentToken()); - if (container instanceof CollectionFieldValue) { - token = buffer.next(); - DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType(); - if (container instanceof WeightedSet) { - // these are objects with string keys (which are the nested - // types) and values which are the weight - WeightedSet weightedSet = (WeightedSet) container; - fillWeightedSetUpdate(initNesting, valueType, weightedSet); - if (op == FieldOperation.REMOVE) { - singleUpdate = FieldUpdate.createRemoveAll(field, weightedSet); - } else { - singleUpdate = FieldUpdate.createAddAll(field, weightedSet); - - } - } else { - List<FieldValue> arrayContents = new ArrayList<>(); - token = fillArrayUpdate(initNesting, token, valueType, arrayContents); - if (token != JsonToken.END_ARRAY) { - throw new IllegalStateException("Expected END_ARRAY. Got '" + token + "'."); - } - if (op == FieldOperation.REMOVE) { - singleUpdate = FieldUpdate.createRemoveAll(field, arrayContents); - } else { - singleUpdate = FieldUpdate.createAddAll(field, arrayContents); - } - } - } else { - throw new UnsupportedOperationException( - "Trying to add or remove from a field of a type the reader does not know how to handle: " - + container.getClass().getName()); - } - expectCompositeEnd(buffer.currentToken()); - update.addAll(singleUpdate); - } - - private JsonToken fillArrayUpdate(int initNesting, JsonToken initToken, DataType valueType, List<FieldValue> arrayContents) { - JsonToken token = initToken; - while (buffer.nesting() >= initNesting) { - arrayContents.add(readSingleValue(token, valueType)); - token = buffer.next(); - } - return token; - } - - private void fillWeightedSetUpdate(int initNesting, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { - iterateThroughWeightedSet(initNesting, valueType, weightedSet); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private void iterateThroughWeightedSet(int initNesting, DataType valueType, WeightedSet weightedSet) { - while (buffer.nesting() >= initNesting) { - // XXX the keys are defined in the spec to always be represented as strings - FieldValue v = valueType.createFieldValue(buffer.currentName()); - weightedSet.put(v, Integer.valueOf(buffer.currentText())); - buffer.next(); - } - } - - // TODO populateComposite is extremely similar to add/remove, refactor - // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders - @SuppressWarnings({ "cast", "rawtypes" }) - private void populateComposite(FieldValue parent, JsonToken token) { - if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) { - throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'."); - } - if (parent instanceof CollectionFieldValue) { - DataType valueType = ((CollectionFieldValue) parent).getDataType().getNestedType(); - if (parent instanceof WeightedSet) { - fillWeightedSet(valueType, (WeightedSet) parent); - } else { - fillArray((CollectionFieldValue) parent, valueType); - } - } else if (parent instanceof MapFieldValue) { - fillMap((MapFieldValue) parent); - } else if (parent instanceof StructuredFieldValue) { - fillStruct((StructuredFieldValue) parent); - } else if (parent instanceof TensorFieldValue) { - fillTensor((TensorFieldValue) parent); - } else { - throw new IllegalStateException("Has created a composite field" - + " value the reader does not know how to handle: " - + parent.getClass().getName() + " This is a bug. token = " + token); - } - expectCompositeEnd(buffer.currentToken()); - } - - private void expectCompositeEnd(JsonToken token) { - Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token); - } - - private void fillStruct(StructuredFieldValue parent) { - // do note the order of initializing initNesting and token is relevant for empty docs - int initNesting = buffer.nesting(); - JsonToken token = buffer.next(); - - while (buffer.nesting() >= initNesting) { - Field f = getField(parent); - try { - FieldValue v = readSingleValue(token, f.getDataType()); - parent.setFieldValue(f, v); - token = buffer.next(); - } catch (IllegalArgumentException e) { - throw new JsonReaderException(f, e); - } - } - } - - private Field getField(StructuredFieldValue parent) { - Field f = parent.getField(buffer.currentName()); - if (f == null) { - throw new NullPointerException("Could not get field \"" + buffer.currentName() + - "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\"."); - } - return f; - } - - @SuppressWarnings({ "rawtypes", "cast", "unchecked" }) - private void fillMap(MapFieldValue parent) { - JsonToken token = buffer.currentToken(); - int initNesting = buffer.nesting(); - expectArrayStart(token); - token = buffer.next(); - DataType keyType = parent.getDataType().getKeyType(); - DataType valueType = parent.getDataType().getValueType(); - while (buffer.nesting() >= initNesting) { - FieldValue key = null; - FieldValue value = null; - expectObjectStart(token); - token = buffer.next(); - for (int i = 0; i < 2; ++i) { - if (MAP_KEY.equals(buffer.currentName())) { - key = readSingleValue(token, keyType); - } else if (MAP_VALUE.equals(buffer.currentName())) { - value = readSingleValue(token, valueType); - } - token = buffer.next(); - } - Preconditions.checkState(key != null && value != null, "Missing key or value for map entry."); - parent.put(key, value); - - expectObjectEnd(token); - token = buffer.next(); // array end or next entry - } - } - - private void expectArrayStart(JsonToken token) { - Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token); - } - - private void expectObjectStart(JsonToken token) { - Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token); - } - - private void expectObjectEnd(JsonToken token) { - Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token); - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - private void fillArray(CollectionFieldValue parent, DataType valueType) { - int initNesting = buffer.nesting(); - expectArrayStart(buffer.currentToken()); - JsonToken token = buffer.next(); - while (buffer.nesting() >= initNesting) { - parent.add(readSingleValue(token, valueType)); - token = buffer.next(); - } - } - - private void fillWeightedSet(DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { - int initNesting = buffer.nesting(); - expectObjectStart(buffer.currentToken()); - buffer.next(); - iterateThroughWeightedSet(initNesting, valueType, weightedSet); - } - - private void fillTensor(TensorFieldValue tensorFieldValue) { - Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); - expectObjectStart(buffer.currentToken()); - int initNesting = buffer.nesting(); - // read tensor cell fields and ignore everything else - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - if (TENSOR_CELLS.equals(buffer.currentName())) - readTensorCells(tensorBuilder); - } - expectObjectEnd(buffer.currentToken()); - tensorFieldValue.assign(tensorBuilder.build()); - } - - private void readTensorCells(Tensor.Builder tensorBuilder) { - expectArrayStart(buffer.currentToken()); - int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) - readTensorCell(tensorBuilder); - expectCompositeEnd(buffer.currentToken()); - } - - private void readTensorCell(Tensor.Builder tensorBuilder) { - expectObjectStart(buffer.currentToken()); - int initNesting = buffer.nesting(); - double cellValue = 0.0; - Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - String currentName = buffer.currentName(); - if (TENSOR_ADDRESS.equals(currentName)) { - readTensorAddress(cellBuilder); - } else if (TENSOR_VALUE.equals(currentName)) { - cellValue = Double.valueOf(buffer.currentText()); - } - } - expectObjectEnd(buffer.currentToken()); - cellBuilder.value(cellValue); - } - - private void readTensorAddress(MappedTensor.Builder.CellBuilder cellBuilder) { - expectObjectStart(buffer.currentToken()); - int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - String dimension = buffer.currentName(); - String label = buffer.currentText(); - cellBuilder.label(dimension, label); - } - expectObjectEnd(buffer.currentToken()); - } - - private FieldValue readSingleValue(JsonToken t, DataType expectedType) { - if (t.isScalarValue()) { - return readAtomic(expectedType); - } else { - FieldValue v = expectedType.createFieldValue(); - populateComposite(v, t); - return v; - } - } - - private FieldValue readAtomic(DataType expectedType) { - if (expectedType.equals(DataType.RAW)) { - return expectedType.createFieldValue(new Base64().decode(buffer.currentText())); - } else if (expectedType.equals(PositionDataType.INSTANCE)) { - return PositionDataType.fromString(buffer.currentText()); - } else if (expectedType instanceof ReferenceDataType) { - return readReferenceFieldValue(expectedType); - } else { - return expectedType.createFieldValue(buffer.currentText()); - } - } - - private FieldValue readReferenceFieldValue(DataType expectedType) { - final FieldValue value = expectedType.createFieldValue(); - final String refText = buffer.currentText(); - if (!refText.isEmpty()) { - value.assign(new DocumentId(buffer.currentText())); - } - return value; - } - - private void bufferFields(JsonToken current) { - buffer.bufferObject(current, parser); - } - - Optional<DocumentParseInfo> parseDocument() { - // we should now be at the start of a feed operation or at the end of the feed - JsonToken token = nextToken(); - if (token == JsonToken.END_ARRAY) { - return Optional.empty(); // end of feed - } - expectObjectStart(token); - - DocumentParseInfo documentParseInfo = new DocumentParseInfo(); - - while (true) { - try { - token = nextToken(); - if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) && - CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { - documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE); - continue; - } - if (token == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) { - documentParseInfo.condition = Optional.of(parser.getText()); - continue; - } - if (token == JsonToken.START_OBJECT) { - try { - if (!FIELDS.equals(parser.getCurrentName())) { - throw new IllegalArgumentException("Unexpected object key: " + parser.getCurrentName()); - } - } catch (IOException e) { - // TODO more specific wrapping - throw new RuntimeException(e); - } - bufferFields(token); - continue; - } - if (token == JsonToken.END_OBJECT) { - if (documentParseInfo.documentId == null) { - throw new RuntimeException("Did not find document operation"); - } - return Optional.of(documentParseInfo); - } - if (token == JsonToken.VALUE_STRING) { - documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName()); - documentParseInfo.documentId = new DocumentId(parser.getText()); - continue; - } - throw new RuntimeException("Expected document start or document operation."); - } catch (IOException e) { - throw new IllegalStateException(e); - } - - } - } - - private static SupportedOperation operationNameToOperationType(String operationName) { - switch (operationName) { - case PUT: - case ID: - return SupportedOperation.PUT; - case REMOVE: - return SupportedOperation.REMOVE; - case UPDATE: - return SupportedOperation.UPDATE; - default: - throw new IllegalArgumentException( - "Got " + operationName + " as document operation, only \"put\", " + - "\"remove\" and \"update\" are supported."); - } - } - - DocumentType readDocumentType(DocumentId docId) { + public DocumentType readDocumentType(DocumentId docId) { return getDocumentTypeFromString(docId.getDocType(), typeManager); } @@ -748,12 +223,13 @@ public class JsonReader { return docType; } - private JsonToken nextToken() { + public static JsonToken nextToken(JsonParser parser) { try { return parser.nextValue(); } catch (IOException e) { // Jackson is not able to recover from structural parse errors - state = ReaderState.END_OF_FEED; + // TODO Do we really need to set state on exception? + // state = ReaderState.END_OF_FEED; throw new RuntimeException(e); } } diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 5fb8948649a..855fabc7a63 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -21,6 +21,7 @@ import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.StructuredFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.readers.TensorReader; import com.yahoo.document.serialization.FieldWriter; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; @@ -34,6 +35,9 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; +import static com.yahoo.document.json.readers.MapReader.MAP_KEY; +import static com.yahoo.document.json.readers.MapReader.MAP_VALUE; + /** * @author Steinar Knutsen * @author Vegard Sjonfjell @@ -78,7 +82,7 @@ public class JsonSerializationHelper { } private static void serializeTensorDimensions(JsonGenerator generator, Set<String> dimensions) throws IOException { - generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS); + generator.writeArrayFieldStart(TensorReader.TENSOR_DIMENSIONS); for (String dimension : dimensions) { generator.writeString(dimension); } @@ -87,11 +91,11 @@ public class JsonSerializationHelper { } private static void serializeTensorCells(JsonGenerator generator, Tensor tensor) throws IOException { - generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS); + generator.writeArrayFieldStart(TensorReader.TENSOR_CELLS); for (Map.Entry<TensorAddress, Double> cell : tensor.cells().entrySet()) { generator.writeStartObject(); serializeTensorAddress(generator, cell.getKey(), tensor.type()); - generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue()); + generator.writeNumberField(TensorReader.TENSOR_VALUE, cell.getValue()); generator.writeEndObject(); } @@ -99,7 +103,7 @@ public class JsonSerializationHelper { } private static void serializeTensorAddress(JsonGenerator generator, TensorAddress address, TensorType type) throws IOException { - generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS); + generator.writeObjectFieldStart(TensorReader.TENSOR_ADDRESS); for (int i = 0; i < type.dimensions().size(); i++) generator.writeStringField(type.dimensions().get(i).name(), address.label(i)); @@ -186,9 +190,9 @@ public class JsonSerializationHelper { for (Map.Entry<K, V> entry : map.entrySet()) { generator.writeStartObject(); - generator.writeFieldName(JsonReader.MAP_KEY); + generator.writeFieldName(MAP_KEY); entry.getKey().serialize(null, fieldWriter); - generator.writeFieldName(JsonReader.MAP_VALUE); + generator.writeFieldName(MAP_VALUE); entry.getValue().serialize(null, fieldWriter); generator.writeEndObject(); } diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index 6ad526a2aaa..bc90141747f 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -38,6 +38,8 @@ import java.util.Iterator; import java.util.Map; import static com.yahoo.document.json.JsonSerializationHelper.*; +import static com.yahoo.document.json.document.DocumentParser.FIELDS; +import static com.yahoo.document.json.document.DocumentParser.REMOVE; /** * Serialize Document and other FieldValue instances as JSON. @@ -106,7 +108,7 @@ public class JsonWriter implements DocumentWriter { // this makes it impossible to refeed directly, not sure what's correct // perhaps just change to "put"? generator.writeStringField("id", value.getId().toString()); - generator.writeObjectFieldStart(JsonReader.FIELDS); + generator.writeObjectFieldStart(FIELDS); Iterator<Map.Entry<Field, FieldValue>> i = value.iterator(); while (i.hasNext()) { @@ -251,7 +253,7 @@ public class JsonWriter implements DocumentWriter { try { JsonGenerator throwAway = jsonFactory.createGenerator(out); throwAway.writeStartObject(); - throwAway.writeStringField(JsonReader.REMOVE, docId.toString()); + throwAway.writeStringField(REMOVE, docId.toString()); throwAway.writeEndObject(); throwAway.close(); } catch (IOException e) { diff --git a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java index 4476acfae22..0ca5ba744f1 100644 --- a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java @@ -6,6 +6,7 @@ import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.json.document.DocumentParser; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import java.io.IOException; @@ -25,14 +26,14 @@ public class SingleDocumentParser { } public VespaXMLFeedReader.Operation parsePut(InputStream inputStream, String docId) { - return parse(inputStream, docId, JsonReader.SupportedOperation.PUT); + return parse(inputStream, docId, DocumentParser.SupportedOperation.PUT); } public VespaXMLFeedReader.Operation parseUpdate(InputStream inputStream, String docId) { - return parse(inputStream, docId, JsonReader.SupportedOperation.UPDATE); + return parse(inputStream, docId, DocumentParser.SupportedOperation.UPDATE); } - private VespaXMLFeedReader.Operation parse(InputStream inputStream, String docId, JsonReader.SupportedOperation supportedOperation) { + private VespaXMLFeedReader.Operation parse(InputStream inputStream, String docId, DocumentParser.SupportedOperation supportedOperation) { final JsonReader reader = new JsonReader(docMan, inputStream, jsonFactory); final DocumentOperation documentOperation = reader.readSingleDocument(supportedOperation, docId); VespaXMLFeedReader.Operation operation = new VespaXMLFeedReader.Operation(); @@ -41,7 +42,7 @@ public class SingleDocumentParser { } catch (IOException e) { throw new RuntimeException(e); } - if (supportedOperation == JsonReader.SupportedOperation.PUT) { + if (supportedOperation == DocumentParser.SupportedOperation.PUT) { operation.setDocument(((DocumentPut) documentOperation).getDocument()); } else { operation.setDocumentUpdate((DocumentUpdate) documentOperation); diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index 8f3395b989e..9f6a0ba3641 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -15,11 +15,11 @@ import com.google.common.base.Preconditions; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ -class TokenBuffer { - static final class Token { - final JsonToken token; - final String name; - final String text; +public class TokenBuffer { + public static final class Token { + public final JsonToken token; + public final String name; + public final String text; Token(JsonToken token, String name, String text) { this.token = token; @@ -31,7 +31,7 @@ class TokenBuffer { private Deque<Token> buffer; private int nesting = 0; - TokenBuffer() { + public TokenBuffer() { this(new ArrayDeque<>()); } @@ -42,7 +42,7 @@ class TokenBuffer { } } - JsonToken next() { + public JsonToken next() { buffer.removeFirst(); Token t = buffer.peekFirst(); if (t == null) { @@ -52,15 +52,15 @@ class TokenBuffer { return t.token; } - JsonToken currentToken() { + public JsonToken currentToken() { return buffer.peekFirst().token; } - String currentName() { + public String currentName() { return buffer.peekFirst().name; } - String currentText() { + public String currentText() { return buffer.peekFirst().text; } @@ -72,7 +72,7 @@ class TokenBuffer { buffer.addLast(new Token(token, name, text)); } - void bufferObject(JsonToken first, JsonParser tokens) { + public void bufferObject(JsonToken first, JsonParser tokens) { int localNesting = 0; JsonToken t = first; @@ -146,7 +146,7 @@ class TokenBuffer { } } - TokenBuffer prefetchCurrentElement() { + public TokenBuffer prefetchCurrentElement() { Deque<Token> copy = new ArrayDeque<>(); if (currentToken().isScalarValue()) { @@ -165,7 +165,7 @@ class TokenBuffer { return new TokenBuffer(copy); } - Token prefetchScalar(String name) { + public Token prefetchScalar(String name) { int localNesting = nesting(); int nestingBarrier = localNesting; Token toReturn = null; diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java new file mode 100644 index 00000000000..e464100a59d --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java @@ -0,0 +1,151 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.document; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.DocumentId; +import com.yahoo.document.json.readers.DocumentParseInfo; + +import java.io.IOException; +import java.util.Optional; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; + +public class DocumentParser { + public enum SupportedOperation { + PUT, UPDATE, REMOVE + } + private static final String UPDATE = "update"; + private static final String PUT = "put"; + private static final String ID = "id"; + private static final String CONDITION = "condition"; + public static final String CREATE_IF_NON_EXISTENT = "create"; + public static final String FIELDS = "fields"; + public static final String REMOVE = "remove"; + + public static Optional<DocumentParseInfo> parseDocument(JsonParser parser) { + // we should now be at the start of a feed operation or at the end of the feed + JsonToken token = nextToken(parser); + if (token == JsonToken.END_ARRAY) { + return Optional.empty(); // end of feed + } + expectObjectStart(token); + + DocumentParseInfo documentParseInfo = new DocumentParseInfo(); + + while (true) { + try { + token = nextToken(parser); + if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) && + CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE); + continue; + } + if (token == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) { + documentParseInfo.condition = Optional.of(parser.getText()); + continue; + } + if (token == JsonToken.START_OBJECT) { + try { + if (!FIELDS.equals(parser.getCurrentName())) { + throw new IllegalArgumentException("Unexpected object key: " + parser.getCurrentName()); + } + } catch (IOException e) { + // TODO more specific wrapping + throw new RuntimeException(e); + } + documentParseInfo.fieldsBuffer.bufferObject(token, parser); + continue; + } + if (token == JsonToken.END_OBJECT) { + if (documentParseInfo.documentId == null) { + throw new RuntimeException("Did not find document operation"); + } + return Optional.of(documentParseInfo); + } + if (token == JsonToken.VALUE_STRING) { + documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName()); + documentParseInfo.documentId = new DocumentId(parser.getText()); + continue; + } + throw new RuntimeException("Expected document start or document operation."); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + } + + private static SupportedOperation operationNameToOperationType(String operationName) { + switch (operationName) { + case PUT: + case ID: + return SupportedOperation.PUT; + case REMOVE: + return SupportedOperation.REMOVE; + case UPDATE: + return SupportedOperation.UPDATE; + default: + throw new IllegalArgumentException( + "Got " + operationName + " as document operation, only \"put\", " + + "\"remove\" and \"update\" are supported."); + } + } + + public static DocumentParseInfo parseDocumentsFields(JsonParser parser, DocumentId documentId) { + long indentLevel = 0; + DocumentParseInfo documentParseInfo = new DocumentParseInfo(); + documentParseInfo.documentId = documentId; + while (true) { + // we should now be at the start of a feed operation or at the end of the feed + JsonToken t = nextToken(parser); + if (t == null) { + throw new IllegalArgumentException("Could not read document, no document?"); + } + switch (t) { + case START_OBJECT: + indentLevel++; + break; + case END_OBJECT: + indentLevel--; + break; + case START_ARRAY: + indentLevel+=10000L; + break; + case END_ARRAY: + indentLevel-=10000L; + break; + } + if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) { + try { + if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue()); + continue; + } + } catch (IOException e) { + throw new RuntimeException("Got IO exception while parsing document", e); + } + } + if (indentLevel == 2L && t == JsonToken.START_OBJECT) { + + try { + if (!FIELDS.equals(parser.getCurrentName())) { + continue; + } + } catch (IOException e) { + throw new RuntimeException("Got IO exception while parsing document", e); + } + documentParseInfo.fieldsBuffer.bufferObject(t, parser); + break; + } + } + return documentParseInfo; + } + + private static JsonToken nextToken(JsonParser parser) { + try { + return parser.nextValue(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java new file mode 100644 index 00000000000..aa79e2dad44 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java @@ -0,0 +1,80 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.base.Preconditions; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.document.update.FieldUpdate; + +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectCompositeEnd; +import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSetUpdate; + +public class AddRemoveCreator { + + // yes, this suppresswarnings ugliness is by intention, the code relies on + // the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) + public static void createAdds(TokenBuffer buffer, Field field, FieldUpdate update) { + createAddsOrRemoves(buffer, field, update, false); + } + + // yes, this suppresswarnings ugliness is by intention, the code relies on + // the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) + public static void createRemoves(TokenBuffer buffer, Field field, FieldUpdate update) { + createAddsOrRemoves(buffer, field, update, true); + } + + // yes, this suppresswarnings ugliness is by intention, the code relies on + // the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) + private static void createAddsOrRemoves(TokenBuffer buffer, Field field, FieldUpdate update, boolean isRemove) { + FieldValue container = field.getDataType().createFieldValue(); + FieldUpdate singleUpdate; + int initNesting = buffer.nesting(); + JsonToken token; + + Preconditions.checkState(buffer.currentToken().isStructStart(), "Expected start of composite, got %s", buffer.currentToken()); + if (container instanceof CollectionFieldValue) { + token = buffer.next(); + DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType(); + if (container instanceof WeightedSet) { + // these are objects with string keys (which are the nested + // types) and values which are the weight + WeightedSet weightedSet = (WeightedSet) container; + fillWeightedSetUpdate(buffer, initNesting, valueType, weightedSet); + if (isRemove) { + singleUpdate = FieldUpdate.createRemoveAll(field, weightedSet); + } else { + singleUpdate = FieldUpdate.createAddAll(field, weightedSet); + + } + } else { + List<FieldValue> arrayContents = new ArrayList<>(); + token = ArrayReader.fillArrayUpdate(buffer, initNesting, token, valueType, arrayContents); + if (token != JsonToken.END_ARRAY) { + throw new IllegalStateException("Expected END_ARRAY. Got '" + token + "'."); + } + if (isRemove) { + singleUpdate = FieldUpdate.createRemoveAll(field, arrayContents); + } else { + singleUpdate = FieldUpdate.createAddAll(field, arrayContents); + } + } + } else { + throw new UnsupportedOperationException( + "Trying to add or remove from a field of a type the reader does not know how to handle: " + + container.getClass().getName()); + } + expectCompositeEnd(buffer.currentToken()); + update.addAll(singleUpdate); + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java new file mode 100644 index 00000000000..ab3c3e23816 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.json.TokenBuffer; + +import java.util.List; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart; +import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; + +public class ArrayReader { + static public JsonToken fillArrayUpdate(TokenBuffer buffer, int initNesting, JsonToken initToken, DataType valueType, List<FieldValue> arrayContents) { + JsonToken token = initToken; + while (buffer.nesting() >= initNesting) { + arrayContents.add(readSingleValue(buffer, token, valueType)); + token = buffer.next(); + } + return token; + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public static void fillArray(TokenBuffer buffer, CollectionFieldValue parent, DataType valueType) { + int initNesting = buffer.nesting(); + expectArrayStart(buffer.currentToken()); + JsonToken token = buffer.next(); + while (buffer.nesting() >= initNesting) { + parent.add(readSingleValue(buffer, token, valueType)); + token = buffer.next(); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java new file mode 100644 index 00000000000..b9f4d1d4613 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java @@ -0,0 +1,60 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.TokenBuffer; + +import static com.yahoo.document.json.readers.ArrayReader.fillArray; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectCompositeEnd; +import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSet; + +public class CompositeReader { + + // TODO createComposite is extremely similar to add/remove, refactor + // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes" }) + public static FieldValue createComposite(TokenBuffer buffer, DataType expectedType) { + FieldValue fieldValue = expectedType.createFieldValue(); + populateComposite(buffer, fieldValue); + return fieldValue; + } + + // TODO createComposite is extremely similar to add/remove, refactor + // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes" }) + public static void populateComposite(TokenBuffer buffer, FieldValue fieldValue) { + JsonToken token = buffer.currentToken(); + if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) { + throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'."); + } + if (fieldValue instanceof CollectionFieldValue) { + DataType valueType = ((CollectionFieldValue) fieldValue).getDataType().getNestedType(); + if (fieldValue instanceof WeightedSet) { + fillWeightedSet(buffer, valueType, (WeightedSet) fieldValue); + } else { + fillArray(buffer, (CollectionFieldValue) fieldValue, valueType); + } + } else if (fieldValue instanceof MapFieldValue) { + MapReader.fillMap(buffer, (MapFieldValue) fieldValue); + } else if (fieldValue instanceof StructuredFieldValue) { + StructReader.fillStruct(buffer, (StructuredFieldValue) fieldValue); + } else if (fieldValue instanceof TensorFieldValue) { + TensorReader.fillTensor(buffer, (TensorFieldValue) fieldValue); + } else { + throw new IllegalStateException("Has created a composite field" + + " value the reader does not know how to handle: " + + fieldValue.getClass().getName() + " This is a bug. token = " + token); + } + expectCompositeEnd(buffer.currentToken()); + } + + + +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java new file mode 100644 index 00000000000..7401e33c607 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.document.json.document.DocumentParser; + +import java.util.Optional; + +public class DocumentParseInfo { + public DocumentId documentId; + public Optional<Boolean> create = Optional.empty(); + public Optional<String> condition = Optional.empty(); + public DocumentParser.SupportedOperation operationType = null; + public TokenBuffer fieldsBuffer = new TokenBuffer(); +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java b/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java new file mode 100644 index 00000000000..75e0860e2d5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java @@ -0,0 +1,23 @@ +package com.yahoo.document.json.readers; + + +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.base.Preconditions; + +public class JsonParserHelpers { + public static void expectArrayStart(JsonToken token) { + Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token); + } + + public static void expectObjectStart(JsonToken token) { + Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token); + } + + public static void expectObjectEnd(JsonToken token) { + Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token); + } + + public static void expectCompositeEnd(JsonToken token) { + Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token); + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/MapReader.java b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java new file mode 100644 index 00000000000..0897f05e82a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java @@ -0,0 +1,126 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.base.Preconditions; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.ValueUpdate; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; +import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate; +import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; + +public class MapReader { + public static final String MAP_KEY = "key"; + public static final String MAP_VALUE = "value"; + public static final String UPDATE_ELEMENT = "element"; + public static final String UPDATE_MATCH = "match"; + + @SuppressWarnings({ "rawtypes", "cast", "unchecked" }) + public static void fillMap(TokenBuffer buffer, MapFieldValue parent) { + JsonToken token = buffer.currentToken(); + int initNesting = buffer.nesting(); + expectArrayStart(token); + token = buffer.next(); + DataType keyType = parent.getDataType().getKeyType(); + DataType valueType = parent.getDataType().getValueType(); + while (buffer.nesting() >= initNesting) { + FieldValue key = null; + FieldValue value = null; + expectObjectStart(token); + token = buffer.next(); + for (int i = 0; i < 2; ++i) { + if (MAP_KEY.equals(buffer.currentName())) { + key = readSingleValue(buffer, token, keyType); + } else if (MAP_VALUE.equals(buffer.currentName())) { + value = readSingleValue(buffer, token, valueType); + } + token = buffer.next(); + } + Preconditions.checkState(key != null && value != null, "Missing key or value for map entry."); + parent.put(key, value); + + expectObjectEnd(token); + token = buffer.next(); // array end or next entry + } + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public static ValueUpdate createMapUpdate(TokenBuffer buffer, DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) { + TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT); + if (UPDATE_ELEMENT.equals(buffer.currentName())) { + buffer.next(); + } + + FieldValue key = keyTypeForMapUpdate(element, currentLevel); + if (keyParent != null) { + ((CollectionFieldValue) keyParent).add(key); + } + // structure is: [(match + element)*, (element + action)] + // match will always have element, and either match or action + if (!UPDATE_MATCH.equals(buffer.currentName())) { + // we have reached an action... + if (topLevelKey == null) { + return ValueUpdate.createMap(key, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName())); + } else { + return ValueUpdate.createMap(topLevelKey, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName())); + } + } else { + // next level of matching + if (topLevelKey == null) { + return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, key); + } else { + return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, topLevelKey); + } + } + } + + @SuppressWarnings("rawtypes") + public static ValueUpdate createMapUpdate(TokenBuffer buffer, Field field) { + buffer.next(); + MapValueUpdate m = (MapValueUpdate) MapReader.createMapUpdate(buffer, field.getDataType(), null, null); + buffer.next(); + // must generate the field value in parallell with the actual + return m; + + } + + private static DataType valueTypeForMapUpdate(DataType parentType) { + if (parentType instanceof WeightedSetDataType) { + return DataType.INT; + } else if (parentType instanceof CollectionDataType) { + return ((CollectionDataType) parentType).getNestedType(); + } else if (parentType instanceof MapDataType) { + return ((MapDataType) parentType).getValueType(); + } else { + throw new UnsupportedOperationException("Unexpected parent type: " + parentType); + } + } + + private static FieldValue keyTypeForMapUpdate(TokenBuffer.Token element, DataType expectedType) { + FieldValue v; + if (expectedType instanceof ArrayDataType) { + v = new IntegerFieldValue(Integer.valueOf(element.text)); + } else if (expectedType instanceof WeightedSetDataType) { + v = ((WeightedSetDataType) expectedType).getNestedType().createFieldValue(element.text); + } else if (expectedType instanceof MapDataType) { + v = ((MapDataType) expectedType).getKeyType().createFieldValue(element.text); + } else { + throw new IllegalArgumentException("Container type " + expectedType + " not supported for match update."); + } + return v; + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java new file mode 100644 index 00000000000..24ad5f0589a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java @@ -0,0 +1,78 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentId; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.document.update.ValueUpdate; +import org.apache.commons.codec.binary.Base64; + +public class SingleValueReader { + public static final String UPDATE_ASSIGN = "assign"; + public static final String UPDATE_INCREMENT = "increment"; + public static final String UPDATE_DECREMENT = "decrement"; + public static final String UPDATE_MULTIPLY = "multiply"; + public static final String UPDATE_DIVIDE = "divide"; + + public static FieldValue readSingleValue(TokenBuffer buffer, JsonToken t, DataType expectedType) { + if (t.isScalarValue()) { + return readAtomic(buffer, expectedType); + } else { + return CompositeReader.createComposite(buffer, expectedType); + } + } + + @SuppressWarnings("rawtypes") + public static ValueUpdate readSingleUpdate(TokenBuffer buffer, DataType expectedType, String action) { + ValueUpdate update; + + switch (action) { + case UPDATE_ASSIGN: + update = (buffer.currentToken() == JsonToken.VALUE_NULL) + ? ValueUpdate.createClear() + : ValueUpdate.createAssign(readSingleValue(buffer, buffer.currentToken(), expectedType)); + break; + // double is silly, but it's what is used internally anyway + case UPDATE_INCREMENT: + update = ValueUpdate.createIncrement(Double.valueOf(buffer.currentText())); + break; + case UPDATE_DECREMENT: + update = ValueUpdate.createDecrement(Double.valueOf(buffer.currentText())); + break; + case UPDATE_MULTIPLY: + update = ValueUpdate.createMultiply(Double.valueOf(buffer.currentText())); + break; + case UPDATE_DIVIDE: + update = ValueUpdate.createDivide(Double.valueOf(buffer.currentText())); + break; + default: + throw new IllegalArgumentException("Operation \"" + buffer.currentName() + "\" not implemented."); + } + return update; + } + + public static FieldValue readAtomic(TokenBuffer buffer, DataType expectedType) { + if (expectedType.equals(DataType.RAW)) { + return expectedType.createFieldValue(new Base64().decode(buffer.currentText())); + } else if (expectedType.equals(PositionDataType.INSTANCE)) { + return PositionDataType.fromString(buffer.currentText()); + } else if (expectedType instanceof ReferenceDataType) { + return readReferenceFieldValue(buffer, expectedType); + } else { + return expectedType.createFieldValue(buffer.currentText()); + } + } + + private static FieldValue readReferenceFieldValue(TokenBuffer buffer, DataType expectedType) { + final FieldValue value = expectedType.createFieldValue(); + final String refText = buffer.currentText(); + if (!refText.isEmpty()) { + value.assign(new DocumentId(buffer.currentText())); + } + return value; + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java new file mode 100644 index 00000000000..17c9123e1cd --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java @@ -0,0 +1,40 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.json.JsonReaderException; +import com.yahoo.document.json.TokenBuffer; + +import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; + +public class StructReader { + public static void fillStruct(TokenBuffer buffer, StructuredFieldValue parent) { + // do note the order of initializing initNesting and token is relevant for empty docs + int initNesting = buffer.nesting(); + JsonToken token = buffer.next(); + + while (buffer.nesting() >= initNesting) { + Field f = getField(buffer, parent); + try { + FieldValue v = readSingleValue(buffer, token, f.getDataType()); + parent.setFieldValue(f, v); + token = buffer.next(); + } catch (IllegalArgumentException e) { + throw new JsonReaderException(f, e); + } + } + } + + public static Field getField(TokenBuffer buffer, StructuredFieldValue parent) { + Field f = parent.getField(buffer.currentName()); + if (f == null) { + throw new NullPointerException("Could not get field \"" + buffer.currentName() + + "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\"."); + } + return f; + } + +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java new file mode 100644 index 00000000000..5dd32753ed6 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -0,0 +1,65 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.tensor.MappedTensor; +import com.yahoo.tensor.Tensor; + +import static com.yahoo.document.json.readers.JsonParserHelpers.*; + +public class TensorReader { + public static final String TENSOR_ADDRESS = "address"; + public static final String TENSOR_DIMENSIONS = "dimensions"; + public static final String TENSOR_CELLS = "cells"; + public static final String TENSOR_VALUE = "value"; + + public static void fillTensor(TokenBuffer buffer, TensorFieldValue tensorFieldValue) { + Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + /* read tensor cell fields and ignore everything else */ + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + if (TensorReader.TENSOR_CELLS.equals(buffer.currentName())) + readTensorCells(buffer, tensorBuilder); + } + expectObjectEnd(buffer.currentToken()); + tensorFieldValue.assign(tensorBuilder.build()); + } + + public static void readTensorCells(TokenBuffer buffer, Tensor.Builder tensorBuilder) { + expectArrayStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) + readTensorCell(buffer, tensorBuilder); + expectCompositeEnd(buffer.currentToken()); + } + + public static void readTensorCell(TokenBuffer buffer, Tensor.Builder tensorBuilder) { + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + double cellValue = 0.0; + Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + String currentName = buffer.currentName(); + if (TensorReader.TENSOR_ADDRESS.equals(currentName)) { + readTensorAddress(buffer, cellBuilder); + } else if (TensorReader.TENSOR_VALUE.equals(currentName)) { + cellValue = Double.valueOf(buffer.currentText()); + } + } + expectObjectEnd(buffer.currentToken()); + cellBuilder.value(cellValue); + } + + public static void readTensorAddress(TokenBuffer buffer, MappedTensor.Builder.CellBuilder cellBuilder) { + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + String dimension = buffer.currentName(); + String label = buffer.currentText(); + cellBuilder.label(dimension, label); + } + expectObjectEnd(buffer.currentToken()); + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java new file mode 100644 index 00000000000..af4f5a8d226 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.TokenBuffer; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; + + +public class WeightedSetReader { + public static void fillWeightedSet(TokenBuffer buffer, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { + int initNesting = buffer.nesting(); + expectObjectStart(buffer.currentToken()); + buffer.next(); + iterateThroughWeightedSet(buffer, initNesting, valueType, weightedSet); + } + public static void fillWeightedSetUpdate(TokenBuffer buffer, int initNesting, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { + iterateThroughWeightedSet(buffer, initNesting, valueType, weightedSet); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static void iterateThroughWeightedSet(TokenBuffer buffer, int initNesting, DataType valueType, WeightedSet weightedSet) { + while (buffer.nesting() >= initNesting) { + // XXX the keys are defined in the spec to always be represented as strings + FieldValue v = valueType.createFieldValue(buffer.currentName()); + weightedSet.put(v, Integer.valueOf(buffer.currentText())); + buffer.next(); + } + } +} |