diff options
author | freva <valerijf@yahoo-inc.com> | 2017-02-09 14:00:57 +0100 |
---|---|---|
committer | freva <valerijf@yahoo-inc.com> | 2017-02-09 14:00:57 +0100 |
commit | b1239de86db85b1cf94d9ff57b50d8bf0a5d29a5 (patch) | |
tree | 24500a7318edecf6b61391de0b65a12477beecf1 /document | |
parent | d9e75e756f7081c14a136f6096356565e173d81e (diff) | |
parent | ee78e9d3af060ee832c9527d09e8f858ae2f7bbf (diff) |
Merge branch 'master' into freva/fieldpath-parsing
# Conflicts:
# document/src/main/java/com/yahoo/document/json/JsonReader.java
Diffstat (limited to 'document')
26 files changed, 913 insertions, 704 deletions
diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java index 359873e1cf4..f486ab6d33d 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUpdate.java +++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java @@ -29,7 +29,7 @@ import java.util.Optional; * docUpdate.addFieldUpdate(update); * </pre> * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @see com.yahoo.document.update.FieldUpdate * @see com.yahoo.document.update.ValueUpdate */ diff --git a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java index 5d65b25499c..1e4079dd3fe 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/ReferenceFieldValue.java @@ -151,4 +151,15 @@ public class ReferenceFieldValue extends FieldValue { public void deserialize(Field field, FieldReader reader) { reader.read(field, this); } + + /** + * Expose target document ID as this value's wrapped value. Primarily implemented to + * allow for transparent interoperability when used in concrete document types. + * + * @return reference DocumentId, or null if none has been set + */ + @Override + public DocumentId getWrappedValue() { + return documentId.orElse(null); + } } diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java index b799a56197f..ef048fd020b 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java @@ -15,9 +15,10 @@ import java.util.HashMap; import java.util.Map; /** - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author Thomas Gundersen */ public class AssignFieldPathUpdate extends FieldPathUpdate { + class SimpleAssignIteratorHandler extends FieldPathIteratorHandler { FieldValue newValue; boolean removeIfZero; @@ -278,4 +279,5 @@ public class AssignFieldPathUpdate extends FieldPathUpdate { public FieldValue getFieldValue() { return fieldValue; } + } diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java index 318b696ce7a..01c36ed5753 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java @@ -14,11 +14,11 @@ import com.yahoo.document.serialization.DocumentUpdateReader; import com.yahoo.document.serialization.VespaDocumentSerializerHead; /** - * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> + * @author Thomas Gundersen */ public abstract class FieldPathUpdate { - public static enum Type { + public enum Type { ASSIGN(0), REMOVE(1), ADD(2); diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index 4af429340e4..7034ff56dfb 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -6,9 +6,6 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.google.common.annotations.Beta; import com.google.common.base.Preconditions; -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; @@ -18,40 +15,27 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; -import com.yahoo.document.MapDataType; -import com.yahoo.document.NumericDataType; -import com.yahoo.document.PositionDataType; -import com.yahoo.document.ReferenceDataType; import com.yahoo.document.TestAndSetCondition; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.document.datatypes.CollectionFieldValue; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.MapFieldValue; -import com.yahoo.document.datatypes.StructuredFieldValue; -import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.document.datatypes.WeightedSet; -import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; -import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; -import com.yahoo.document.fieldpathupdate.FieldPathUpdate; -import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; -import com.yahoo.document.json.TokenBuffer.Token; -import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.json.document.DocumentParser; +import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.update.FieldUpdate; -import com.yahoo.document.update.MapValueUpdate; -import com.yahoo.document.update.ValueUpdate; -import com.yahoo.tensor.MappedTensor; -import com.yahoo.tensor.Tensor; -import org.apache.commons.codec.binary.Base64; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.Optional; +import static com.yahoo.document.json.JsonReader.ReaderState.END_OF_FEED; +import static com.yahoo.document.json.document.DocumentParser.parseDocumentsFields; +import static com.yahoo.document.json.readers.AddRemoveCreator.createAdds; +import static com.yahoo.document.json.readers.AddRemoveCreator.createRemoves; +import static com.yahoo.document.json.readers.CompositeReader.populateComposite; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; +import static com.yahoo.document.json.readers.MapReader.UPDATE_MATCH; +import static com.yahoo.document.json.readers.MapReader.createMapUpdate; +import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate; + /** * Initialize Vespa documents/updates/removes from an InputStream containing a * valid JSON representation of a feed. @@ -62,60 +46,18 @@ import java.util.Optional; @Beta public class JsonReader { - private enum FieldOperation { - ADD, REMOVE + // Only used for testing. + public Optional<DocumentParseInfo> parseDocument() throws IOException { + return DocumentParser.parseDocument(parser); } - static final String MAP_KEY = "key"; - static final String MAP_VALUE = "value"; - static final String FIELDS = "fields"; - static final String FIELDPATHS = "fieldpaths"; - static final String REMOVE = "remove"; - static final String UPDATE_INCREMENT = "increment"; - static final String UPDATE_DECREMENT = "decrement"; - static final String UPDATE_MULTIPLY = "multiply"; - static final String UPDATE_DIVIDE = "divide"; - static final String TENSOR_DIMENSIONS = "dimensions"; - static final String TENSOR_CELLS = "cells"; - static final String TENSOR_ADDRESS = "address"; - static final String TENSOR_VALUE = "value"; - - private static final String UPDATE = "update"; - private static final String PUT = "put"; - private static final String ID = "id"; - private static final String CONDITION = "condition"; - private static final String CREATE_IF_NON_EXISTENT = "create"; - private static final String UPDATE_ASSIGN = "assign"; private static final String UPDATE_REMOVE = "remove"; - private static final String UPDATE_MATCH = "match"; private static final String UPDATE_ADD = "add"; - private static final String UPDATE_ELEMENT = "element"; private final JsonParser parser; - private TokenBuffer buffer = new TokenBuffer(); private final DocumentTypeManager typeManager; private ReaderState state = ReaderState.AT_START; - static class DocumentParseInfo { - public DocumentId documentId; - public Optional<Boolean> create = Optional.empty(); - Optional<String> condition = Optional.empty(); - SupportedOperation operationType = null; - private List<FieldPathUpdate> fieldPathUpdatess = new ArrayList<>(); - - public void addFieldPathUpdates(FieldPathUpdate fieldPath) { - fieldPathUpdatess.add(fieldPath); - } - - public List<FieldPathUpdate> getFieldPathUpdates() { - return fieldPathUpdatess; - } - } - - enum SupportedOperation { - PUT, UPDATE, REMOVE - } - enum ReaderState { AT_START, READING, END_OF_FEED } @@ -126,7 +68,7 @@ public class JsonReader { try { parser = parserFactory.createParser(input); } catch (IOException e) { - state = ReaderState.END_OF_FEED; + state = END_OF_FEED; throw new RuntimeException(e); } } @@ -137,12 +79,17 @@ public class JsonReader { * @param docIdString document ID. * @return the document */ - public DocumentOperation readSingleDocument(SupportedOperation operationType, String docIdString) { + public DocumentOperation readSingleDocument(DocumentParser.SupportedOperation operationType, String docIdString) { DocumentId docId = new DocumentId(docIdString); - DocumentType documentType = getDocumentTypeFromString(docId.getDocType(), typeManager); - DocumentParseInfo documentParseInfo = parseToDocumentsFieldsAndInsertFieldsIntoBuffer(docId, documentType); + DocumentParseInfo documentParseInfo = null; + try { + documentParseInfo = parseDocumentsFields(parser, docId); + } catch (IOException e) { + state = END_OF_FEED; + throw new RuntimeException(e); + } documentParseInfo.operationType = operationType; - DocumentOperation operation = createDocumentOperation(documentType, documentParseInfo); + DocumentOperation operation = createDocumentOperation(documentParseInfo.fieldsBuffer, documentParseInfo); operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.condition)); return operation; } @@ -150,7 +97,7 @@ public class JsonReader { public DocumentOperation next() { switch (state) { case AT_START: - JsonToken t = nextToken(); + JsonToken t = nextToken(parser); expectArrayStart(t); state = ReaderState.READING; break; @@ -159,41 +106,40 @@ public class JsonReader { case READING: break; } - - Optional<DocumentParseInfo> documentParseInfo = parseDocument(); - + Optional<DocumentParseInfo> documentParseInfo; + try { + documentParseInfo = DocumentParser.parseDocument(parser); + } catch (IOException r) { + // Jackson is not able to recover from structural parse errors + state = END_OF_FEED; + throw new RuntimeException(r); + } if (! documentParseInfo.isPresent()) { - state = ReaderState.END_OF_FEED; + state = END_OF_FEED; return null; } - - DocumentType documentType = getDocumentTypeFromString(documentParseInfo.get().documentId.getDocType(), typeManager); - DocumentOperation operation = createDocumentOperation(documentType, documentParseInfo.get()); + DocumentOperation operation = createDocumentOperation(documentParseInfo.get().fieldsBuffer, documentParseInfo.get()); operation.setCondition(TestAndSetCondition.fromConditionString(documentParseInfo.get().condition)); return operation; } - private DocumentOperation createDocumentOperation(DocumentType documentType, DocumentParseInfo documentParseInfo) { + private DocumentOperation createDocumentOperation(TokenBuffer buffer, DocumentParseInfo documentParseInfo) { + DocumentType documentType = getDocumentTypeFromString(documentParseInfo.documentId.getDocType(), typeManager); final DocumentOperation documentOperation; try { switch (documentParseInfo.operationType) { case PUT: documentOperation = new DocumentPut(new Document(documentType, documentParseInfo.documentId)); - readPut((DocumentPut) documentOperation); - verifyEndState(); + readPut(buffer, (DocumentPut) documentOperation); + verifyEndState(buffer); break; case REMOVE: documentOperation = new DocumentRemove(documentParseInfo.documentId); break; case UPDATE: documentOperation = new DocumentUpdate(documentType, documentParseInfo.documentId); - documentParseInfo.getFieldPathUpdates().forEach( - ((DocumentUpdate) documentOperation)::addFieldPathUpdate); - - if (buffer.size() > 0) { - readUpdate((DocumentUpdate) documentOperation); - verifyEndState(); - } + readUpdate(buffer, (DocumentUpdate) documentOperation); + verifyEndState(buffer); break; default: throw new IllegalStateException("Implementation out of sync with itself. This is a bug."); @@ -211,79 +157,34 @@ public class JsonReader { return documentOperation; } - void readUpdate(DocumentUpdate next) { + // Exposed for unit testing... + void readUpdate(TokenBuffer buffer, DocumentUpdate next) { if (buffer.size() == 0) { - bufferFields(nextToken()); + buffer.bufferObject(nextToken(parser), parser); } - populateUpdateFromBuffer(next); + populateUpdateFromBuffer(buffer, next); } - void readPut(DocumentPut put) { + // Exposed for unit testing... + void readPut(TokenBuffer buffer, DocumentPut put) { if (buffer.size() == 0) { - bufferFields(nextToken()); + buffer.bufferObject(nextToken(parser), parser); } - JsonToken t = buffer.currentToken(); try { - populateComposite(put.getDocument(), t); + populateComposite(buffer, put.getDocument()); } catch (JsonReaderException e) { throw JsonReaderException.addDocId(e, put.getId()); } } - private DocumentParseInfo parseToDocumentsFieldsAndInsertFieldsIntoBuffer(DocumentId documentId, DocumentType documentType) { - DocumentParseInfo documentParseInfo = new DocumentParseInfo(); - documentParseInfo.documentId = documentId; - - while (true) { - // we should now be at the start of a feed operation or at the end of the feed - JsonToken t = nextToken(); - if (t == null) break; - - try { - if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { - documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue()); - - } else if (FIELDS.equals(parser.getCurrentName())) { - bufferFields(t); - - } else if (FIELDPATHS.equals(parser.getCurrentName())) { - expectArrayStart(parser.currentToken()); - parser.nextToken(); - do { - FieldPathUpdate fieldPathUpdate = parseFieldPathUpdate(documentType, parser); - documentParseInfo.addFieldPathUpdates(fieldPathUpdate); - } while (parser.nextToken() != JsonToken.END_ARRAY); - } - } catch (IOException e) { - throw new RuntimeException("Got IO exception while parsing document", e); - } catch (ParseException e) { - throw new RuntimeException("Failed to parse document", e); - } - } - return documentParseInfo; - } - - private static FieldPathUpdate parseFieldPathUpdate(DocumentType documentType, JsonParser parser) throws IOException, ParseException { - assert parser.isExpectedStartObjectToken(); - - parser.nextToken(); - FieldPathUpdate.Type operation = FieldPathUpdate.Type.valueOf(parser.getValueAsString().toUpperCase()); - parser.nextToken(); - - VespaJsonDocumentReader jsonDocumentReader = new VespaJsonDocumentReader(parser); - FieldPathUpdate fieldPathUpdate = FieldPathUpdate.create(operation, documentType, jsonDocumentReader); - - return fieldPathUpdate; - } - - private void verifyEndState() { + private void verifyEndState(TokenBuffer buffer) { Preconditions.checkState(buffer.nesting() == 0, "Nesting not zero at end of operation"); expectObjectEnd(buffer.currentToken()); Preconditions.checkState(buffer.next() == null, "Dangling data at end of operation"); Preconditions.checkState(buffer.size() == 0, "Dangling data at end of operation"); } - private void populateUpdateFromBuffer(DocumentUpdate update) { + private static void populateUpdateFromBuffer(TokenBuffer buffer, DocumentUpdate update) { expectObjectStart(buffer.currentToken()); int localNesting = buffer.nesting(); JsonToken t = buffer.next(); @@ -292,12 +193,12 @@ public class JsonReader { expectObjectStart(t); String fieldName = buffer.currentName(); Field field = update.getType().getField(fieldName); - addFieldUpdates(update, field); + addFieldUpdates(buffer, update, field); t = buffer.next(); } } - private void addFieldUpdates(DocumentUpdate update, Field field) { + private static void addFieldUpdates(TokenBuffer buffer, DocumentUpdate update, Field field) { int localNesting = buffer.nesting(); FieldUpdate fieldUpdate = FieldUpdate.create(field); @@ -305,457 +206,24 @@ public class JsonReader { while (localNesting <= buffer.nesting()) { switch (buffer.currentName()) { case UPDATE_REMOVE: - createAddsOrRemoves(field, fieldUpdate, FieldOperation.REMOVE); + createRemoves(buffer, field, fieldUpdate); break; case UPDATE_ADD: - createAddsOrRemoves(field, fieldUpdate, FieldOperation.ADD); + createAdds(buffer, field, fieldUpdate); break; case UPDATE_MATCH: - fieldUpdate.addValueUpdate(createMapUpdate(field)); + fieldUpdate.addValueUpdate(createMapUpdate(buffer, field)); break; default: String action = buffer.currentName(); - fieldUpdate.addValueUpdate(readSingleUpdate(field.getDataType(), action)); + fieldUpdate.addValueUpdate(readSingleUpdate(buffer, field.getDataType(), action)); } buffer.next(); } update.addFieldUpdate(fieldUpdate); } - @SuppressWarnings("rawtypes") - private ValueUpdate createMapUpdate(Field field) { - buffer.next(); - MapValueUpdate m = (MapValueUpdate) createMapUpdate(field.getDataType(), null, null); - buffer.next(); - // must generate the field value in parallell with the actual - return m; - - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private ValueUpdate createMapUpdate(DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) { - TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT); - if (UPDATE_ELEMENT.equals(buffer.currentName())) { - buffer.next(); - } - - FieldValue key = keyTypeForMapUpdate(element, currentLevel); - if (keyParent != null) { - ((CollectionFieldValue) keyParent).add(key); - } - // structure is: [(match + element)*, (element + action)] - // match will always have element, and either match or action - if (!UPDATE_MATCH.equals(buffer.currentName())) { - // we have reached an action... - if (topLevelKey == null) { - return ValueUpdate.createMap(key, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName())); - } else { - return ValueUpdate.createMap(topLevelKey, readSingleUpdate(valueTypeForMapUpdate(currentLevel), buffer.currentName())); - } - } else { - // next level of matching - if (topLevelKey == null) { - return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, key); - } else { - return createMapUpdate(valueTypeForMapUpdate(currentLevel), key, topLevelKey); - } - } - } - - private DataType valueTypeForMapUpdate(DataType parentType) { - if (parentType instanceof WeightedSetDataType) { - return DataType.INT; - } else if (parentType instanceof CollectionDataType) { - return ((CollectionDataType) parentType).getNestedType(); - } else if (parentType instanceof MapDataType) { - return ((MapDataType) parentType).getValueType(); - } else { - throw new UnsupportedOperationException("Unexpected parent type: " + parentType); - } - } - - private FieldValue keyTypeForMapUpdate(Token element, DataType expectedType) { - FieldValue v; - if (expectedType instanceof ArrayDataType) { - v = new IntegerFieldValue(Integer.valueOf(element.text)); - } else if (expectedType instanceof WeightedSetDataType) { - v = ((WeightedSetDataType) expectedType).getNestedType().createFieldValue(element.text); - } else if (expectedType instanceof MapDataType) { - v = ((MapDataType) expectedType).getKeyType().createFieldValue(element.text); - } else { - throw new IllegalArgumentException("Container type " + expectedType + " not supported for match update."); - } - return v; - } - - @SuppressWarnings("rawtypes") - private ValueUpdate readSingleUpdate(DataType expectedType, String action) { - ValueUpdate update; - - switch (action) { - case UPDATE_ASSIGN: - update = (buffer.currentToken() == JsonToken.VALUE_NULL) - ? ValueUpdate.createClear() - : ValueUpdate.createAssign(readSingleValue(buffer.currentToken(), expectedType)); - break; - // double is silly, but it's what is used internally anyway - case UPDATE_INCREMENT: - update = ValueUpdate.createIncrement(Double.valueOf(buffer.currentText())); - break; - case UPDATE_DECREMENT: - update = ValueUpdate.createDecrement(Double.valueOf(buffer.currentText())); - break; - case UPDATE_MULTIPLY: - update = ValueUpdate.createMultiply(Double.valueOf(buffer.currentText())); - break; - case UPDATE_DIVIDE: - update = ValueUpdate.createDivide(Double.valueOf(buffer.currentText())); - break; - default: - throw new IllegalArgumentException("Operation \"" + buffer.currentName() + "\" not implemented."); - } - return update; - } - - // yes, this suppresswarnings ugliness is by intention, the code relies on - // the contracts in the builders - @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) - private void createAddsOrRemoves(Field field, FieldUpdate update, FieldOperation op) { - FieldValue container = field.getDataType().createFieldValue(); - FieldUpdate singleUpdate; - int initNesting = buffer.nesting(); - JsonToken token; - - Preconditions.checkState(buffer.currentToken().isStructStart(), "Expected start of composite, got %s", buffer.currentToken()); - if (container instanceof CollectionFieldValue) { - token = buffer.next(); - DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType(); - if (container instanceof WeightedSet) { - // these are objects with string keys (which are the nested - // types) and values which are the weight - WeightedSet weightedSet = (WeightedSet) container; - fillWeightedSetUpdate(initNesting, valueType, weightedSet); - if (op == FieldOperation.REMOVE) { - singleUpdate = FieldUpdate.createRemoveAll(field, weightedSet); - } else { - singleUpdate = FieldUpdate.createAddAll(field, weightedSet); - - } - } else { - List<FieldValue> arrayContents = new ArrayList<>(); - token = fillArrayUpdate(initNesting, token, valueType, arrayContents); - if (token != JsonToken.END_ARRAY) { - throw new IllegalStateException("Expected END_ARRAY. Got '" + token + "'."); - } - if (op == FieldOperation.REMOVE) { - singleUpdate = FieldUpdate.createRemoveAll(field, arrayContents); - } else { - singleUpdate = FieldUpdate.createAddAll(field, arrayContents); - } - } - } else { - throw new UnsupportedOperationException( - "Trying to add or remove from a field of a type the reader does not know how to handle: " - + container.getClass().getName()); - } - expectCompositeEnd(buffer.currentToken()); - update.addAll(singleUpdate); - } - - private JsonToken fillArrayUpdate(int initNesting, JsonToken initToken, DataType valueType, List<FieldValue> arrayContents) { - JsonToken token = initToken; - while (buffer.nesting() >= initNesting) { - arrayContents.add(readSingleValue(token, valueType)); - token = buffer.next(); - } - return token; - } - - private void fillWeightedSetUpdate(int initNesting, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { - iterateThroughWeightedSet(initNesting, valueType, weightedSet); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private void iterateThroughWeightedSet(int initNesting, DataType valueType, WeightedSet weightedSet) { - while (buffer.nesting() >= initNesting) { - // XXX the keys are defined in the spec to always be represented as strings - FieldValue v = valueType.createFieldValue(buffer.currentName()); - weightedSet.put(v, Integer.valueOf(buffer.currentText())); - buffer.next(); - } - } - - // TODO populateComposite is extremely similar to add/remove, refactor - // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders - @SuppressWarnings({ "cast", "rawtypes" }) - private void populateComposite(FieldValue parent, JsonToken token) { - if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) { - throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'."); - } - if (parent instanceof CollectionFieldValue) { - DataType valueType = ((CollectionFieldValue) parent).getDataType().getNestedType(); - if (parent instanceof WeightedSet) { - fillWeightedSet(valueType, (WeightedSet) parent); - } else { - fillArray((CollectionFieldValue) parent, valueType); - } - } else if (parent instanceof MapFieldValue) { - fillMap((MapFieldValue) parent); - } else if (parent instanceof StructuredFieldValue) { - fillStruct((StructuredFieldValue) parent); - } else if (parent instanceof TensorFieldValue) { - fillTensor((TensorFieldValue) parent); - } else { - throw new IllegalStateException("Has created a composite field" - + " value the reader does not know how to handle: " - + parent.getClass().getName() + " This is a bug. token = " + token); - } - expectCompositeEnd(buffer.currentToken()); - } - - private void expectCompositeEnd(JsonToken token) { - Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token); - } - - private void fillStruct(StructuredFieldValue parent) { - // do note the order of initializing initNesting and token is relevant for empty docs - int initNesting = buffer.nesting(); - JsonToken token = buffer.next(); - - while (buffer.nesting() >= initNesting) { - Field f = getField(parent); - try { - FieldValue v = readSingleValue(token, f.getDataType()); - parent.setFieldValue(f, v); - token = buffer.next(); - } catch (IllegalArgumentException e) { - throw new JsonReaderException(f, e); - } - } - } - - private Field getField(StructuredFieldValue parent) { - Field f = parent.getField(buffer.currentName()); - if (f == null) { - throw new NullPointerException("Could not get field \"" + buffer.currentName() + - "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\"."); - } - return f; - } - - @SuppressWarnings({ "rawtypes", "cast", "unchecked" }) - private void fillMap(MapFieldValue parent) { - JsonToken token = buffer.currentToken(); - int initNesting = buffer.nesting(); - expectArrayStart(token); - token = buffer.next(); - DataType keyType = parent.getDataType().getKeyType(); - DataType valueType = parent.getDataType().getValueType(); - while (buffer.nesting() >= initNesting) { - FieldValue key = null; - FieldValue value = null; - expectObjectStart(token); - token = buffer.next(); - for (int i = 0; i < 2; ++i) { - if (MAP_KEY.equals(buffer.currentName())) { - key = readSingleValue(token, keyType); - } else if (MAP_VALUE.equals(buffer.currentName())) { - value = readSingleValue(token, valueType); - } - token = buffer.next(); - } - Preconditions.checkState(key != null && value != null, "Missing key or value for map entry."); - parent.put(key, value); - - expectObjectEnd(token); - token = buffer.next(); // array end or next entry - } - } - - private void expectArrayStart(JsonToken token) { - Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token); - } - - private void expectObjectStart(JsonToken token) { - Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token); - } - - private void expectObjectEnd(JsonToken token) { - Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token); - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - private void fillArray(CollectionFieldValue parent, DataType valueType) { - int initNesting = buffer.nesting(); - expectArrayStart(buffer.currentToken()); - JsonToken token = buffer.next(); - while (buffer.nesting() >= initNesting) { - parent.add(readSingleValue(token, valueType)); - token = buffer.next(); - } - } - - private void fillWeightedSet(DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { - int initNesting = buffer.nesting(); - expectObjectStart(buffer.currentToken()); - buffer.next(); - iterateThroughWeightedSet(initNesting, valueType, weightedSet); - } - - private void fillTensor(TensorFieldValue tensorFieldValue) { - Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); - expectObjectStart(buffer.currentToken()); - int initNesting = buffer.nesting(); - // read tensor cell fields and ignore everything else - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - if (TENSOR_CELLS.equals(buffer.currentName())) - readTensorCells(tensorBuilder); - } - expectObjectEnd(buffer.currentToken()); - tensorFieldValue.assign(tensorBuilder.build()); - } - - private void readTensorCells(Tensor.Builder tensorBuilder) { - expectArrayStart(buffer.currentToken()); - int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) - readTensorCell(tensorBuilder); - expectCompositeEnd(buffer.currentToken()); - } - - private void readTensorCell(Tensor.Builder tensorBuilder) { - expectObjectStart(buffer.currentToken()); - int initNesting = buffer.nesting(); - double cellValue = 0.0; - Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - String currentName = buffer.currentName(); - if (TENSOR_ADDRESS.equals(currentName)) { - readTensorAddress(cellBuilder); - } else if (TENSOR_VALUE.equals(currentName)) { - cellValue = Double.valueOf(buffer.currentText()); - } - } - expectObjectEnd(buffer.currentToken()); - cellBuilder.value(cellValue); - } - - private void readTensorAddress(MappedTensor.Builder.CellBuilder cellBuilder) { - expectObjectStart(buffer.currentToken()); - int initNesting = buffer.nesting(); - for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { - String dimension = buffer.currentName(); - String label = buffer.currentText(); - cellBuilder.label(dimension, label); - } - expectObjectEnd(buffer.currentToken()); - } - - private FieldValue readSingleValue(JsonToken t, DataType expectedType) { - if (t.isScalarValue()) { - return readAtomic(expectedType); - } else { - FieldValue v = expectedType.createFieldValue(); - populateComposite(v, t); - return v; - } - } - - private FieldValue readAtomic(DataType expectedType) { - if (expectedType.equals(DataType.RAW)) { - return expectedType.createFieldValue(new Base64().decode(buffer.currentText())); - } else if (expectedType.equals(PositionDataType.INSTANCE)) { - return PositionDataType.fromString(buffer.currentText()); - } else if (expectedType instanceof ReferenceDataType) { - return readReferenceFieldValue(expectedType); - } else { - return expectedType.createFieldValue(buffer.currentText()); - } - } - - private FieldValue readReferenceFieldValue(DataType expectedType) { - final FieldValue value = expectedType.createFieldValue(); - final String refText = buffer.currentText(); - if (!refText.isEmpty()) { - value.assign(new DocumentId(buffer.currentText())); - } - return value; - } - - private void bufferFields(JsonToken current) { - buffer.bufferObject(current, parser); - } - - Optional<DocumentParseInfo> parseDocument() { - // we should now be at the start of a feed operation or at the end of the feed - JsonToken token = nextToken(); - if (token == JsonToken.END_ARRAY) { - return Optional.empty(); // end of feed - } - expectObjectStart(token); - - DocumentParseInfo documentParseInfo = new DocumentParseInfo(); - - while (true) { - try { - token = nextToken(); - if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) && - CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { - documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE); - continue; - } - if (token == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) { - documentParseInfo.condition = Optional.of(parser.getText()); - continue; - } - if (token == JsonToken.START_OBJECT) { - try { - if (!FIELDS.equals(parser.getCurrentName())) { - throw new IllegalArgumentException("Unexpected object key: " + parser.getCurrentName()); - } - } catch (IOException e) { - // TODO more specific wrapping - throw new RuntimeException(e); - } - bufferFields(token); - continue; - } - if (token == JsonToken.END_OBJECT) { - if (documentParseInfo.documentId == null) { - throw new RuntimeException("Did not find document operation"); - } - return Optional.of(documentParseInfo); - } - if (token == JsonToken.VALUE_STRING) { - documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName()); - documentParseInfo.documentId = new DocumentId(parser.getText()); - continue; - } - throw new RuntimeException("Expected document start or document operation."); - } catch (IOException e) { - throw new IllegalStateException(e); - } - - } - } - - private static SupportedOperation operationNameToOperationType(String operationName) { - switch (operationName) { - case PUT: - case ID: - return SupportedOperation.PUT; - case REMOVE: - return SupportedOperation.REMOVE; - case UPDATE: - return SupportedOperation.UPDATE; - default: - throw new IllegalArgumentException( - "Got " + operationName + " as document operation, only \"put\", " + - "\"remove\" and \"update\" are supported."); - } - } - - DocumentType readDocumentType(DocumentId docId) { + public DocumentType readDocumentType(DocumentId docId) { return getDocumentTypeFromString(docId.getDocType(), typeManager); } @@ -767,12 +235,12 @@ public class JsonReader { return docType; } - private JsonToken nextToken() { + public JsonToken nextToken(JsonParser parser) { try { return parser.nextValue(); } catch (IOException e) { // Jackson is not able to recover from structural parse errors - state = ReaderState.END_OF_FEED; + state = END_OF_FEED; throw new RuntimeException(e); } } diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 5fb8948649a..855fabc7a63 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -21,6 +21,7 @@ import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.StructuredFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.readers.TensorReader; import com.yahoo.document.serialization.FieldWriter; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; @@ -34,6 +35,9 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; +import static com.yahoo.document.json.readers.MapReader.MAP_KEY; +import static com.yahoo.document.json.readers.MapReader.MAP_VALUE; + /** * @author Steinar Knutsen * @author Vegard Sjonfjell @@ -78,7 +82,7 @@ public class JsonSerializationHelper { } private static void serializeTensorDimensions(JsonGenerator generator, Set<String> dimensions) throws IOException { - generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS); + generator.writeArrayFieldStart(TensorReader.TENSOR_DIMENSIONS); for (String dimension : dimensions) { generator.writeString(dimension); } @@ -87,11 +91,11 @@ public class JsonSerializationHelper { } private static void serializeTensorCells(JsonGenerator generator, Tensor tensor) throws IOException { - generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS); + generator.writeArrayFieldStart(TensorReader.TENSOR_CELLS); for (Map.Entry<TensorAddress, Double> cell : tensor.cells().entrySet()) { generator.writeStartObject(); serializeTensorAddress(generator, cell.getKey(), tensor.type()); - generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue()); + generator.writeNumberField(TensorReader.TENSOR_VALUE, cell.getValue()); generator.writeEndObject(); } @@ -99,7 +103,7 @@ public class JsonSerializationHelper { } private static void serializeTensorAddress(JsonGenerator generator, TensorAddress address, TensorType type) throws IOException { - generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS); + generator.writeObjectFieldStart(TensorReader.TENSOR_ADDRESS); for (int i = 0; i < type.dimensions().size(); i++) generator.writeStringField(type.dimensions().get(i).name(), address.label(i)); @@ -186,9 +190,9 @@ public class JsonSerializationHelper { for (Map.Entry<K, V> entry : map.entrySet()) { generator.writeStartObject(); - generator.writeFieldName(JsonReader.MAP_KEY); + generator.writeFieldName(MAP_KEY); entry.getKey().serialize(null, fieldWriter); - generator.writeFieldName(JsonReader.MAP_VALUE); + generator.writeFieldName(MAP_VALUE); entry.getValue().serialize(null, fieldWriter); generator.writeEndObject(); } diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index 6ad526a2aaa..bc90141747f 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -38,6 +38,8 @@ import java.util.Iterator; import java.util.Map; import static com.yahoo.document.json.JsonSerializationHelper.*; +import static com.yahoo.document.json.document.DocumentParser.FIELDS; +import static com.yahoo.document.json.document.DocumentParser.REMOVE; /** * Serialize Document and other FieldValue instances as JSON. @@ -106,7 +108,7 @@ public class JsonWriter implements DocumentWriter { // this makes it impossible to refeed directly, not sure what's correct // perhaps just change to "put"? generator.writeStringField("id", value.getId().toString()); - generator.writeObjectFieldStart(JsonReader.FIELDS); + generator.writeObjectFieldStart(FIELDS); Iterator<Map.Entry<Field, FieldValue>> i = value.iterator(); while (i.hasNext()) { @@ -251,7 +253,7 @@ public class JsonWriter implements DocumentWriter { try { JsonGenerator throwAway = jsonFactory.createGenerator(out); throwAway.writeStartObject(); - throwAway.writeStringField(JsonReader.REMOVE, docId.toString()); + throwAway.writeStringField(REMOVE, docId.toString()); throwAway.writeEndObject(); throwAway.close(); } catch (IOException e) { diff --git a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java index 4476acfae22..0ca5ba744f1 100644 --- a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java @@ -6,6 +6,7 @@ import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.json.document.DocumentParser; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import java.io.IOException; @@ -25,14 +26,14 @@ public class SingleDocumentParser { } public VespaXMLFeedReader.Operation parsePut(InputStream inputStream, String docId) { - return parse(inputStream, docId, JsonReader.SupportedOperation.PUT); + return parse(inputStream, docId, DocumentParser.SupportedOperation.PUT); } public VespaXMLFeedReader.Operation parseUpdate(InputStream inputStream, String docId) { - return parse(inputStream, docId, JsonReader.SupportedOperation.UPDATE); + return parse(inputStream, docId, DocumentParser.SupportedOperation.UPDATE); } - private VespaXMLFeedReader.Operation parse(InputStream inputStream, String docId, JsonReader.SupportedOperation supportedOperation) { + private VespaXMLFeedReader.Operation parse(InputStream inputStream, String docId, DocumentParser.SupportedOperation supportedOperation) { final JsonReader reader = new JsonReader(docMan, inputStream, jsonFactory); final DocumentOperation documentOperation = reader.readSingleDocument(supportedOperation, docId); VespaXMLFeedReader.Operation operation = new VespaXMLFeedReader.Operation(); @@ -41,7 +42,7 @@ public class SingleDocumentParser { } catch (IOException e) { throw new RuntimeException(e); } - if (supportedOperation == JsonReader.SupportedOperation.PUT) { + if (supportedOperation == DocumentParser.SupportedOperation.PUT) { operation.setDocument(((DocumentPut) documentOperation).getDocument()); } else { operation.setDocumentUpdate((DocumentUpdate) documentOperation); diff --git a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java index 8f3395b989e..9f6a0ba3641 100644 --- a/document/src/main/java/com/yahoo/document/json/TokenBuffer.java +++ b/document/src/main/java/com/yahoo/document/json/TokenBuffer.java @@ -15,11 +15,11 @@ import com.google.common.base.Preconditions; * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> */ -class TokenBuffer { - static final class Token { - final JsonToken token; - final String name; - final String text; +public class TokenBuffer { + public static final class Token { + public final JsonToken token; + public final String name; + public final String text; Token(JsonToken token, String name, String text) { this.token = token; @@ -31,7 +31,7 @@ class TokenBuffer { private Deque<Token> buffer; private int nesting = 0; - TokenBuffer() { + public TokenBuffer() { this(new ArrayDeque<>()); } @@ -42,7 +42,7 @@ class TokenBuffer { } } - JsonToken next() { + public JsonToken next() { buffer.removeFirst(); Token t = buffer.peekFirst(); if (t == null) { @@ -52,15 +52,15 @@ class TokenBuffer { return t.token; } - JsonToken currentToken() { + public JsonToken currentToken() { return buffer.peekFirst().token; } - String currentName() { + public String currentName() { return buffer.peekFirst().name; } - String currentText() { + public String currentText() { return buffer.peekFirst().text; } @@ -72,7 +72,7 @@ class TokenBuffer { buffer.addLast(new Token(token, name, text)); } - void bufferObject(JsonToken first, JsonParser tokens) { + public void bufferObject(JsonToken first, JsonParser tokens) { int localNesting = 0; JsonToken t = first; @@ -146,7 +146,7 @@ class TokenBuffer { } } - TokenBuffer prefetchCurrentElement() { + public TokenBuffer prefetchCurrentElement() { Deque<Token> copy = new ArrayDeque<>(); if (currentToken().isScalarValue()) { @@ -165,7 +165,7 @@ class TokenBuffer { return new TokenBuffer(copy); } - Token prefetchScalar(String name) { + public Token prefetchScalar(String name) { int localNesting = nesting(); int nestingBarrier = localNesting; Token toReturn = null; diff --git a/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java new file mode 100644 index 00000000000..8053246a266 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/document/DocumentParser.java @@ -0,0 +1,143 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.document; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.DocumentId; +import com.yahoo.document.json.readers.DocumentParseInfo; + +import java.io.IOException; +import java.util.Optional; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; + +public class DocumentParser { + public enum SupportedOperation { + PUT, UPDATE, REMOVE + } + private static final String UPDATE = "update"; + private static final String PUT = "put"; + private static final String ID = "id"; + private static final String CONDITION = "condition"; + public static final String CREATE_IF_NON_EXISTENT = "create"; + public static final String FIELDS = "fields"; + public static final String REMOVE = "remove"; + + public static Optional<DocumentParseInfo> parseDocument(JsonParser parser) throws IOException { + // we should now be at the start of a feed operation or at the end of the feed + JsonToken token = parser.nextValue(); + if (token == JsonToken.END_ARRAY) { + return Optional.empty(); // end of feed + } + expectObjectStart(token); + + DocumentParseInfo documentParseInfo = new DocumentParseInfo(); + + while (true) { + try { + token = parser.nextValue(); + if ((token == JsonToken.VALUE_TRUE || token == JsonToken.VALUE_FALSE) && + CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + documentParseInfo.create = Optional.of(token == JsonToken.VALUE_TRUE); + continue; + } + if (token == JsonToken.VALUE_STRING && CONDITION.equals(parser.getCurrentName())) { + documentParseInfo.condition = Optional.of(parser.getText()); + continue; + } + if (token == JsonToken.START_OBJECT) { + try { + if (!FIELDS.equals(parser.getCurrentName())) { + throw new IllegalArgumentException("Unexpected object key: " + parser.getCurrentName()); + } + } catch (IOException e) { + // TODO more specific wrapping + throw new RuntimeException(e); + } + documentParseInfo.fieldsBuffer.bufferObject(token, parser); + continue; + } + if (token == JsonToken.END_OBJECT) { + if (documentParseInfo.documentId == null) { + throw new RuntimeException("Did not find document operation"); + } + return Optional.of(documentParseInfo); + } + if (token == JsonToken.VALUE_STRING) { + documentParseInfo.operationType = operationNameToOperationType(parser.getCurrentName()); + documentParseInfo.documentId = new DocumentId(parser.getText()); + continue; + } + throw new RuntimeException("Expected document start or document operation."); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + } + + private static SupportedOperation operationNameToOperationType(String operationName) { + switch (operationName) { + case PUT: + case ID: + return SupportedOperation.PUT; + case REMOVE: + return SupportedOperation.REMOVE; + case UPDATE: + return SupportedOperation.UPDATE; + default: + throw new IllegalArgumentException( + "Got " + operationName + " as document operation, only \"put\", " + + "\"remove\" and \"update\" are supported."); + } + } + + public static DocumentParseInfo parseDocumentsFields(JsonParser parser, DocumentId documentId) throws IOException { + long indentLevel = 0; + DocumentParseInfo documentParseInfo = new DocumentParseInfo(); + documentParseInfo.documentId = documentId; + while (true) { + // we should now be at the start of a feed operation or at the end of the feed + JsonToken t = parser.nextValue(); + if (t == null) { + throw new IllegalArgumentException("Could not read document, no document?"); + } + switch (t) { + case START_OBJECT: + indentLevel++; + break; + case END_OBJECT: + indentLevel--; + break; + case START_ARRAY: + indentLevel+=10000L; + break; + case END_ARRAY: + indentLevel-=10000L; + break; + } + if (indentLevel == 1 && (t == JsonToken.VALUE_TRUE || t == JsonToken.VALUE_FALSE)) { + try { + if (CREATE_IF_NON_EXISTENT.equals(parser.getCurrentName())) { + documentParseInfo.create = Optional.ofNullable(parser.getBooleanValue()); + continue; + } + } catch (IOException e) { + throw new RuntimeException("Got IO exception while parsing document", e); + } + } + if (indentLevel == 2L && t == JsonToken.START_OBJECT) { + + try { + if (!FIELDS.equals(parser.getCurrentName())) { + continue; + } + } catch (IOException e) { + throw new RuntimeException("Got IO exception while parsing document", e); + } + documentParseInfo.fieldsBuffer.bufferObject(t, parser); + break; + } + } + return documentParseInfo; + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java new file mode 100644 index 00000000000..aa79e2dad44 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/AddRemoveCreator.java @@ -0,0 +1,80 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.base.Preconditions; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.document.update.FieldUpdate; + +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectCompositeEnd; +import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSetUpdate; + +public class AddRemoveCreator { + + // yes, this suppresswarnings ugliness is by intention, the code relies on + // the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) + public static void createAdds(TokenBuffer buffer, Field field, FieldUpdate update) { + createAddsOrRemoves(buffer, field, update, false); + } + + // yes, this suppresswarnings ugliness is by intention, the code relies on + // the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) + public static void createRemoves(TokenBuffer buffer, Field field, FieldUpdate update) { + createAddsOrRemoves(buffer, field, update, true); + } + + // yes, this suppresswarnings ugliness is by intention, the code relies on + // the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes", "unchecked" }) + private static void createAddsOrRemoves(TokenBuffer buffer, Field field, FieldUpdate update, boolean isRemove) { + FieldValue container = field.getDataType().createFieldValue(); + FieldUpdate singleUpdate; + int initNesting = buffer.nesting(); + JsonToken token; + + Preconditions.checkState(buffer.currentToken().isStructStart(), "Expected start of composite, got %s", buffer.currentToken()); + if (container instanceof CollectionFieldValue) { + token = buffer.next(); + DataType valueType = ((CollectionFieldValue) container).getDataType().getNestedType(); + if (container instanceof WeightedSet) { + // these are objects with string keys (which are the nested + // types) and values which are the weight + WeightedSet weightedSet = (WeightedSet) container; + fillWeightedSetUpdate(buffer, initNesting, valueType, weightedSet); + if (isRemove) { + singleUpdate = FieldUpdate.createRemoveAll(field, weightedSet); + } else { + singleUpdate = FieldUpdate.createAddAll(field, weightedSet); + + } + } else { + List<FieldValue> arrayContents = new ArrayList<>(); + token = ArrayReader.fillArrayUpdate(buffer, initNesting, token, valueType, arrayContents); + if (token != JsonToken.END_ARRAY) { + throw new IllegalStateException("Expected END_ARRAY. Got '" + token + "'."); + } + if (isRemove) { + singleUpdate = FieldUpdate.createRemoveAll(field, arrayContents); + } else { + singleUpdate = FieldUpdate.createAddAll(field, arrayContents); + } + } + } else { + throw new UnsupportedOperationException( + "Trying to add or remove from a field of a type the reader does not know how to handle: " + + container.getClass().getName()); + } + expectCompositeEnd(buffer.currentToken()); + update.addAll(singleUpdate); + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java new file mode 100644 index 00000000000..ab3c3e23816 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/ArrayReader.java @@ -0,0 +1,35 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.json.TokenBuffer; + +import java.util.List; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart; +import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; + +public class ArrayReader { + static public JsonToken fillArrayUpdate(TokenBuffer buffer, int initNesting, JsonToken initToken, DataType valueType, List<FieldValue> arrayContents) { + JsonToken token = initToken; + while (buffer.nesting() >= initNesting) { + arrayContents.add(readSingleValue(buffer, token, valueType)); + token = buffer.next(); + } + return token; + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public static void fillArray(TokenBuffer buffer, CollectionFieldValue parent, DataType valueType) { + int initNesting = buffer.nesting(); + expectArrayStart(buffer.currentToken()); + JsonToken token = buffer.next(); + while (buffer.nesting() >= initNesting) { + parent.add(readSingleValue(buffer, token, valueType)); + token = buffer.next(); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java new file mode 100644 index 00000000000..b9f4d1d4613 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java @@ -0,0 +1,60 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.TokenBuffer; + +import static com.yahoo.document.json.readers.ArrayReader.fillArray; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectCompositeEnd; +import static com.yahoo.document.json.readers.WeightedSetReader.fillWeightedSet; + +public class CompositeReader { + + // TODO createComposite is extremely similar to add/remove, refactor + // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes" }) + public static FieldValue createComposite(TokenBuffer buffer, DataType expectedType) { + FieldValue fieldValue = expectedType.createFieldValue(); + populateComposite(buffer, fieldValue); + return fieldValue; + } + + // TODO createComposite is extremely similar to add/remove, refactor + // yes, this suppresswarnings ugliness is by intention, the code relies on the contracts in the builders + @SuppressWarnings({ "cast", "rawtypes" }) + public static void populateComposite(TokenBuffer buffer, FieldValue fieldValue) { + JsonToken token = buffer.currentToken(); + if ((token != JsonToken.START_OBJECT) && (token != JsonToken.START_ARRAY)) { + throw new IllegalArgumentException("Expected '[' or '{'. Got '" + token + "'."); + } + if (fieldValue instanceof CollectionFieldValue) { + DataType valueType = ((CollectionFieldValue) fieldValue).getDataType().getNestedType(); + if (fieldValue instanceof WeightedSet) { + fillWeightedSet(buffer, valueType, (WeightedSet) fieldValue); + } else { + fillArray(buffer, (CollectionFieldValue) fieldValue, valueType); + } + } else if (fieldValue instanceof MapFieldValue) { + MapReader.fillMap(buffer, (MapFieldValue) fieldValue); + } else if (fieldValue instanceof StructuredFieldValue) { + StructReader.fillStruct(buffer, (StructuredFieldValue) fieldValue); + } else if (fieldValue instanceof TensorFieldValue) { + TensorReader.fillTensor(buffer, (TensorFieldValue) fieldValue); + } else { + throw new IllegalStateException("Has created a composite field" + + " value the reader does not know how to handle: " + + fieldValue.getClass().getName() + " This is a bug. token = " + token); + } + expectCompositeEnd(buffer.currentToken()); + } + + + +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java new file mode 100644 index 00000000000..7401e33c607 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/DocumentParseInfo.java @@ -0,0 +1,16 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.document.json.document.DocumentParser; + +import java.util.Optional; + +public class DocumentParseInfo { + public DocumentId documentId; + public Optional<Boolean> create = Optional.empty(); + public Optional<String> condition = Optional.empty(); + public DocumentParser.SupportedOperation operationType = null; + public TokenBuffer fieldsBuffer = new TokenBuffer(); +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java b/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java new file mode 100644 index 00000000000..75e0860e2d5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/JsonParserHelpers.java @@ -0,0 +1,23 @@ +package com.yahoo.document.json.readers; + + +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.base.Preconditions; + +public class JsonParserHelpers { + public static void expectArrayStart(JsonToken token) { + Preconditions.checkState(token == JsonToken.START_ARRAY, "Expected start of array, got %s", token); + } + + public static void expectObjectStart(JsonToken token) { + Preconditions.checkState(token == JsonToken.START_OBJECT, "Expected start of JSON object, got %s", token); + } + + public static void expectObjectEnd(JsonToken token) { + Preconditions.checkState(token == JsonToken.END_OBJECT, "Expected end of JSON object, got %s", token); + } + + public static void expectCompositeEnd(JsonToken token) { + Preconditions.checkState(token.isStructEnd(), "Expected end of composite, got %s", token); + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/MapReader.java b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java new file mode 100644 index 00000000000..0897f05e82a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/MapReader.java @@ -0,0 +1,126 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.google.common.base.Preconditions; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.ValueUpdate; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectArrayStart; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectEnd; +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; +import static com.yahoo.document.json.readers.SingleValueReader.readSingleUpdate; +import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; + +public class MapReader { + public static final String MAP_KEY = "key"; + public static final String MAP_VALUE = "value"; + public static final String UPDATE_ELEMENT = "element"; + public static final String UPDATE_MATCH = "match"; + + @SuppressWarnings({ "rawtypes", "cast", "unchecked" }) + public static void fillMap(TokenBuffer buffer, MapFieldValue parent) { + JsonToken token = buffer.currentToken(); + int initNesting = buffer.nesting(); + expectArrayStart(token); + token = buffer.next(); + DataType keyType = parent.getDataType().getKeyType(); + DataType valueType = parent.getDataType().getValueType(); + while (buffer.nesting() >= initNesting) { + FieldValue key = null; + FieldValue value = null; + expectObjectStart(token); + token = buffer.next(); + for (int i = 0; i < 2; ++i) { + if (MAP_KEY.equals(buffer.currentName())) { + key = readSingleValue(buffer, token, keyType); + } else if (MAP_VALUE.equals(buffer.currentName())) { + value = readSingleValue(buffer, token, valueType); + } + token = buffer.next(); + } + Preconditions.checkState(key != null && value != null, "Missing key or value for map entry."); + parent.put(key, value); + + expectObjectEnd(token); + token = buffer.next(); // array end or next entry + } + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public static ValueUpdate createMapUpdate(TokenBuffer buffer, DataType currentLevel, FieldValue keyParent, FieldValue topLevelKey) { + TokenBuffer.Token element = buffer.prefetchScalar(UPDATE_ELEMENT); + if (UPDATE_ELEMENT.equals(buffer.currentName())) { + buffer.next(); + } + + FieldValue key = keyTypeForMapUpdate(element, currentLevel); + if (keyParent != null) { + ((CollectionFieldValue) keyParent).add(key); + } + // structure is: [(match + element)*, (element + action)] + // match will always have element, and either match or action + if (!UPDATE_MATCH.equals(buffer.currentName())) { + // we have reached an action... + if (topLevelKey == null) { + return ValueUpdate.createMap(key, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName())); + } else { + return ValueUpdate.createMap(topLevelKey, readSingleUpdate(buffer, valueTypeForMapUpdate(currentLevel), buffer.currentName())); + } + } else { + // next level of matching + if (topLevelKey == null) { + return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, key); + } else { + return createMapUpdate(buffer, valueTypeForMapUpdate(currentLevel), key, topLevelKey); + } + } + } + + @SuppressWarnings("rawtypes") + public static ValueUpdate createMapUpdate(TokenBuffer buffer, Field field) { + buffer.next(); + MapValueUpdate m = (MapValueUpdate) MapReader.createMapUpdate(buffer, field.getDataType(), null, null); + buffer.next(); + // must generate the field value in parallell with the actual + return m; + + } + + private static DataType valueTypeForMapUpdate(DataType parentType) { + if (parentType instanceof WeightedSetDataType) { + return DataType.INT; + } else if (parentType instanceof CollectionDataType) { + return ((CollectionDataType) parentType).getNestedType(); + } else if (parentType instanceof MapDataType) { + return ((MapDataType) parentType).getValueType(); + } else { + throw new UnsupportedOperationException("Unexpected parent type: " + parentType); + } + } + + private static FieldValue keyTypeForMapUpdate(TokenBuffer.Token element, DataType expectedType) { + FieldValue v; + if (expectedType instanceof ArrayDataType) { + v = new IntegerFieldValue(Integer.valueOf(element.text)); + } else if (expectedType instanceof WeightedSetDataType) { + v = ((WeightedSetDataType) expectedType).getNestedType().createFieldValue(element.text); + } else if (expectedType instanceof MapDataType) { + v = ((MapDataType) expectedType).getKeyType().createFieldValue(element.text); + } else { + throw new IllegalArgumentException("Container type " + expectedType + " not supported for match update."); + } + return v; + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java new file mode 100644 index 00000000000..24ad5f0589a --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/SingleValueReader.java @@ -0,0 +1,78 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentId; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.document.update.ValueUpdate; +import org.apache.commons.codec.binary.Base64; + +public class SingleValueReader { + public static final String UPDATE_ASSIGN = "assign"; + public static final String UPDATE_INCREMENT = "increment"; + public static final String UPDATE_DECREMENT = "decrement"; + public static final String UPDATE_MULTIPLY = "multiply"; + public static final String UPDATE_DIVIDE = "divide"; + + public static FieldValue readSingleValue(TokenBuffer buffer, JsonToken t, DataType expectedType) { + if (t.isScalarValue()) { + return readAtomic(buffer, expectedType); + } else { + return CompositeReader.createComposite(buffer, expectedType); + } + } + + @SuppressWarnings("rawtypes") + public static ValueUpdate readSingleUpdate(TokenBuffer buffer, DataType expectedType, String action) { + ValueUpdate update; + + switch (action) { + case UPDATE_ASSIGN: + update = (buffer.currentToken() == JsonToken.VALUE_NULL) + ? ValueUpdate.createClear() + : ValueUpdate.createAssign(readSingleValue(buffer, buffer.currentToken(), expectedType)); + break; + // double is silly, but it's what is used internally anyway + case UPDATE_INCREMENT: + update = ValueUpdate.createIncrement(Double.valueOf(buffer.currentText())); + break; + case UPDATE_DECREMENT: + update = ValueUpdate.createDecrement(Double.valueOf(buffer.currentText())); + break; + case UPDATE_MULTIPLY: + update = ValueUpdate.createMultiply(Double.valueOf(buffer.currentText())); + break; + case UPDATE_DIVIDE: + update = ValueUpdate.createDivide(Double.valueOf(buffer.currentText())); + break; + default: + throw new IllegalArgumentException("Operation \"" + buffer.currentName() + "\" not implemented."); + } + return update; + } + + public static FieldValue readAtomic(TokenBuffer buffer, DataType expectedType) { + if (expectedType.equals(DataType.RAW)) { + return expectedType.createFieldValue(new Base64().decode(buffer.currentText())); + } else if (expectedType.equals(PositionDataType.INSTANCE)) { + return PositionDataType.fromString(buffer.currentText()); + } else if (expectedType instanceof ReferenceDataType) { + return readReferenceFieldValue(buffer, expectedType); + } else { + return expectedType.createFieldValue(buffer.currentText()); + } + } + + private static FieldValue readReferenceFieldValue(TokenBuffer buffer, DataType expectedType) { + final FieldValue value = expectedType.createFieldValue(); + final String refText = buffer.currentText(); + if (!refText.isEmpty()) { + value.assign(new DocumentId(buffer.currentText())); + } + return value; + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/StructReader.java b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java new file mode 100644 index 00000000000..17c9123e1cd --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/StructReader.java @@ -0,0 +1,40 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.json.JsonReaderException; +import com.yahoo.document.json.TokenBuffer; + +import static com.yahoo.document.json.readers.SingleValueReader.readSingleValue; + +public class StructReader { + public static void fillStruct(TokenBuffer buffer, StructuredFieldValue parent) { + // do note the order of initializing initNesting and token is relevant for empty docs + int initNesting = buffer.nesting(); + JsonToken token = buffer.next(); + + while (buffer.nesting() >= initNesting) { + Field f = getField(buffer, parent); + try { + FieldValue v = readSingleValue(buffer, token, f.getDataType()); + parent.setFieldValue(f, v); + token = buffer.next(); + } catch (IllegalArgumentException e) { + throw new JsonReaderException(f, e); + } + } + } + + public static Field getField(TokenBuffer buffer, StructuredFieldValue parent) { + Field f = parent.getField(buffer.currentName()); + if (f == null) { + throw new NullPointerException("Could not get field \"" + buffer.currentName() + + "\" in the structure of type \"" + parent.getDataType().getDataTypeName() + "\"."); + } + return f; + } + +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java new file mode 100644 index 00000000000..5dd32753ed6 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -0,0 +1,65 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.json.TokenBuffer; +import com.yahoo.tensor.MappedTensor; +import com.yahoo.tensor.Tensor; + +import static com.yahoo.document.json.readers.JsonParserHelpers.*; + +public class TensorReader { + public static final String TENSOR_ADDRESS = "address"; + public static final String TENSOR_DIMENSIONS = "dimensions"; + public static final String TENSOR_CELLS = "cells"; + public static final String TENSOR_VALUE = "value"; + + public static void fillTensor(TokenBuffer buffer, TensorFieldValue tensorFieldValue) { + Tensor.Builder tensorBuilder = Tensor.Builder.of(tensorFieldValue.getDataType().getTensorType()); + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + /* read tensor cell fields and ignore everything else */ + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + if (TensorReader.TENSOR_CELLS.equals(buffer.currentName())) + readTensorCells(buffer, tensorBuilder); + } + expectObjectEnd(buffer.currentToken()); + tensorFieldValue.assign(tensorBuilder.build()); + } + + public static void readTensorCells(TokenBuffer buffer, Tensor.Builder tensorBuilder) { + expectArrayStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) + readTensorCell(buffer, tensorBuilder); + expectCompositeEnd(buffer.currentToken()); + } + + public static void readTensorCell(TokenBuffer buffer, Tensor.Builder tensorBuilder) { + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + double cellValue = 0.0; + Tensor.Builder.CellBuilder cellBuilder = tensorBuilder.cell(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + String currentName = buffer.currentName(); + if (TensorReader.TENSOR_ADDRESS.equals(currentName)) { + readTensorAddress(buffer, cellBuilder); + } else if (TensorReader.TENSOR_VALUE.equals(currentName)) { + cellValue = Double.valueOf(buffer.currentText()); + } + } + expectObjectEnd(buffer.currentToken()); + cellBuilder.value(cellValue); + } + + public static void readTensorAddress(TokenBuffer buffer, MappedTensor.Builder.CellBuilder cellBuilder) { + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + String dimension = buffer.currentName(); + String label = buffer.currentText(); + cellBuilder.label(dimension, label); + } + expectObjectEnd(buffer.currentToken()); + } +} diff --git a/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java new file mode 100644 index 00000000000..af4f5a8d226 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/WeightedSetReader.java @@ -0,0 +1,32 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.TokenBuffer; + +import static com.yahoo.document.json.readers.JsonParserHelpers.expectObjectStart; + + +public class WeightedSetReader { + public static void fillWeightedSet(TokenBuffer buffer, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { + int initNesting = buffer.nesting(); + expectObjectStart(buffer.currentToken()); + buffer.next(); + iterateThroughWeightedSet(buffer, initNesting, valueType, weightedSet); + } + public static void fillWeightedSetUpdate(TokenBuffer buffer, int initNesting, DataType valueType, @SuppressWarnings("rawtypes") WeightedSet weightedSet) { + iterateThroughWeightedSet(buffer, initNesting, valueType, weightedSet); + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static void iterateThroughWeightedSet(TokenBuffer buffer, int initNesting, DataType valueType, WeightedSet weightedSet) { + while (buffer.nesting() >= initNesting) { + // XXX the keys are defined in the spec to always be represented as strings + FieldValue v = valueType.createFieldValue(buffer.currentName()); + weightedSet.put(v, Integer.valueOf(buffer.currentText())); + buffer.next(); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java index 83dc9df2826..89d98571394 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java @@ -40,7 +40,7 @@ public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer42 { for (int i = 0; i < size; i++) { int type = getByte(null); update.addFieldPathUpdate(FieldPathUpdate.create(FieldPathUpdate.Type.valueOf(type), - update.getDocumentType(), this)); + update.getDocumentType(), this)); } } catch (ParseException e) { throw new DeserializationException(e); diff --git a/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java b/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java index 37b4329c934..edb40c6959b 100644 --- a/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java +++ b/document/src/main/java/com/yahoo/document/update/MapValueUpdate.java @@ -25,7 +25,7 @@ import com.yahoo.document.serialization.DocumentUpdateWriter; * must represent a legal operation on an integer value.</li> * </ul> * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class MapValueUpdate extends ValueUpdate { protected FieldValue value; diff --git a/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java index daa65d6e95e..a1d69238463 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/ReferenceFieldValueTestCase.java @@ -13,6 +13,7 @@ import org.junit.rules.ExpectedException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -189,4 +190,16 @@ public class ReferenceFieldValueTestCase { value.assign(newId); } + @Test + public void exposed_wrapped_value_is_null_for_empty_reference() { + ReferenceFieldValue nullRef = new ReferenceFieldValue(referenceTypeFoo()); + assertNull(nullRef.getWrappedValue()); + } + + @Test + public void expose_wrapped_value_is_doc_id_for_non_empty_reference() { + ReferenceFieldValue idRef = new ReferenceFieldValue(referenceTypeFoo(), docId("id:ns:foo::toad")); + assertEquals(docId("id:ns:foo::toad"), idRef.getWrappedValue()); + } + } diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index 5dd3abf1e02..055354757f1 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -13,6 +13,7 @@ import com.yahoo.document.PositionDataType; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.json.document.DocumentParser; import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; import org.junit.Test; @@ -64,7 +65,7 @@ public class DocumentUpdateJsonSerializerTest { private static DocumentUpdate deSerializeDocumentUpdate(String jsonDoc, String docId) { final InputStream rawDoc = new ByteArrayInputStream(Utf8.toBytes(jsonDoc)); JsonReader reader = new JsonReader(types, rawDoc, parserFactory); - return (DocumentUpdate) reader.readSingleDocument(JsonReader.SupportedOperation.UPDATE, docId); + return (DocumentUpdate) reader.readSingleDocument(DocumentParser.SupportedOperation.UPDATE, docId); } private static String serializeDocumentUpdate(DocumentUpdate update) { diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 34c93580eb3..756e3fe3aa9 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -30,6 +30,8 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.json.document.DocumentParser; +import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.document.update.AddValueUpdate; import com.yahoo.document.update.ArithmeticValueUpdate; import com.yahoo.document.update.ArithmeticValueUpdate.Operator; @@ -52,6 +54,7 @@ import org.junit.rules.ExpectedException; import org.mockito.internal.matchers.Contains; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; @@ -63,6 +66,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import static com.yahoo.document.json.readers.SingleValueReader.*; import static com.yahoo.test.json.JsonTestHelper.inputJson; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; @@ -170,7 +174,7 @@ public class JsonReaderTestCase { + " \"fields\": { \"something\": \"smoketest\"," + " \"nalle\": \"bamse\"}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - DocumentPut put = (DocumentPut) r.readSingleDocument(JsonReader.SupportedOperation.PUT, "id:unittest:smoke::whee"); + DocumentPut put = (DocumentPut) r.readSingleDocument(DocumentParser.SupportedOperation.PUT, "id:unittest:smoke::whee"); smokeTestDoc(put.getDocument()); } @@ -181,7 +185,7 @@ public class JsonReaderTestCase { + " \"fields\": { \"something\": {" + " \"assign\": \"orOther\" }}" + " }")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(JsonReader.SupportedOperation.UPDATE, "id:unittest:smoke::whee"); + DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(DocumentParser.SupportedOperation.UPDATE, "id:unittest:smoke::whee"); FieldUpdate f = doc.getFieldUpdate("something"); assertEquals(1, f.size()); assertTrue(f.getValueUpdate(0) instanceof AssignValueUpdate); @@ -194,7 +198,7 @@ public class JsonReaderTestCase { + " \"fields\": { \"int1\": {" + " \"assign\": null }}" + " }")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(JsonReader.SupportedOperation.UPDATE, "id:unittest:smoke::whee"); + DocumentUpdate doc = (DocumentUpdate) r.readSingleDocument(DocumentParser.SupportedOperation.UPDATE, "id:unittest:smoke::whee"); FieldUpdate f = doc.getFieldUpdate("int1"); assertEquals(1, f.size()); assertTrue(f.getValueUpdate(0) instanceof ClearValueUpdate); @@ -203,21 +207,21 @@ public class JsonReaderTestCase { @Test - public final void smokeTest() { + public final void smokeTest() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\"," + " \"fields\": { \"something\": \"smoketest\"," + " \"nalle\": \"bamse\"}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); smokeTestDoc(put.getDocument()); } @Test - public final void docIdLookaheadTest() { + public final void docIdLookaheadTest() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{" + " \"fields\": { \"something\": \"smoketest\"," @@ -225,29 +229,29 @@ public class JsonReaderTestCase { + "\"put\": \"id:unittest:smoke::whee\"" + "}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); smokeTestDoc(put.getDocument()); } @Test - public final void emptyDocTest() { + public final void emptyDocTest() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\"," + " \"fields\": {}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); assertEquals("id:unittest:smoke::whee", parseInfo.documentId.toString()); } @Test - public final void testStruct() { + public final void testStruct() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:mirrors::whee\"," + " \"fields\": { " @@ -255,10 +259,10 @@ public class JsonReaderTestCase { + "\"sandra\": \"person\"," + " \"cloud\": \"another person\"}}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("skuggsjaa")); assertSame(Struct.class, f.getClass()); @@ -267,7 +271,7 @@ public class JsonReaderTestCase { } @Test - public final void testUpdateArray() { + public final void testUpdateArray() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"update\": \"id:unittest:testarray::whee\"," + " \"fields\": { " + "\"actualarray\": {" @@ -275,15 +279,15 @@ public class JsonReaderTestCase { + " \"person\"," + " \"another person\"]}}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId); - r.readUpdate(doc); + r.readUpdate(parseInfo.fieldsBuffer, doc); checkSimpleArrayAdd(doc); } @Test - public final void testUpdateWeighted() { + public final void testUpdateWeighted() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\"," + " \"fields\": { " + "\"actualset\": {" @@ -291,10 +295,10 @@ public class JsonReaderTestCase { + " \"person\": 37," + " \"another person\": 41}}}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId); - r.readUpdate(doc); + r.readUpdate(parseInfo.fieldsBuffer, doc); Map<String, Integer> weights = new HashMap<>(); FieldUpdate x = doc.getFieldUpdate("actualset"); for (ValueUpdate<?> v : x.getValueUpdates()) { @@ -312,7 +316,7 @@ public class JsonReaderTestCase { } @Test - public final void testUpdateMatch() { + public final void testUpdateMatch() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\"," + " \"fields\": { " + "\"actualset\": {" @@ -320,11 +324,11 @@ public class JsonReaderTestCase { + " \"element\": \"person\"," + " \"increment\": 13}}}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId); - r.readUpdate(doc); + r.readUpdate(parseInfo.fieldsBuffer, doc); Map<String, Tuple2<Number, String>> matches = new HashMap<>(); FieldUpdate x = doc.getFieldUpdate("actualset"); for (ValueUpdate<?> v : x.getValueUpdates()) { @@ -344,15 +348,15 @@ public class JsonReaderTestCase { @SuppressWarnings({ "cast", "unchecked", "rawtypes" }) @Test - public final void testArithmeticOperators() { + public final void testArithmeticOperators() throws IOException { Tuple2[] operations = new Tuple2[] { - new Tuple2<String, Operator>(JsonReader.UPDATE_DECREMENT, + new Tuple2<String, Operator>(UPDATE_DECREMENT, ArithmeticValueUpdate.Operator.SUB), - new Tuple2<String, Operator>(JsonReader.UPDATE_DIVIDE, + new Tuple2<String, Operator>(UPDATE_DIVIDE, ArithmeticValueUpdate.Operator.DIV), - new Tuple2<String, Operator>(JsonReader.UPDATE_INCREMENT, + new Tuple2<String, Operator>(UPDATE_INCREMENT, ArithmeticValueUpdate.Operator.ADD), - new Tuple2<String, Operator>(JsonReader.UPDATE_MULTIPLY, + new Tuple2<String, Operator>(UPDATE_MULTIPLY, ArithmeticValueUpdate.Operator.MUL) }; for (Tuple2<String, Operator> operator : operations) { InputStream rawDoc = new ByteArrayInputStream( @@ -362,11 +366,11 @@ public class JsonReaderTestCase { + " \"" + (String) operator.first + "\": 13}}}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId); - r.readUpdate(doc); + r.readUpdate(parseInfo.fieldsBuffer, doc); Map<String, Tuple2<Number, Operator>> matches = new HashMap<>(); FieldUpdate x = doc.getFieldUpdate("actualset"); for (ValueUpdate v : x.getValueUpdates()) { @@ -388,7 +392,7 @@ public class JsonReaderTestCase { @SuppressWarnings("rawtypes") @Test - public final void testArrayIndexing() { + public final void testArrayIndexing() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"update\": \"id:unittest:testarray::whee\"," + " \"fields\": { " + "\"actualarray\": {" @@ -396,11 +400,11 @@ public class JsonReaderTestCase { + " \"element\": 3," + " \"assign\": \"nalle\"}}}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId); - r.readUpdate(doc); + r.readUpdate(parseInfo.fieldsBuffer, doc); Map<Number, String> matches = new HashMap<>(); FieldUpdate x = doc.getFieldUpdate("actualarray"); for (ValueUpdate v : x.getValueUpdates()) { @@ -427,17 +431,17 @@ public class JsonReaderTestCase { } @Test - public final void testWeightedSet() { + public final void testWeightedSet() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testset::whee\"," + " \"fields\": { \"actualset\": {" + " \"nalle\": 2," + " \"tralle\": 7 }}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("actualset")); assertSame(WeightedSet.class, f.getClass()); @@ -448,17 +452,17 @@ public class JsonReaderTestCase { } @Test - public final void testArray() { + public final void testArray() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testarray::whee\"," + " \"fields\": { \"actualarray\": [" + " \"nalle\"," + " \"tralle\"]}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("actualarray")); assertSame(Array.class, f.getClass()); @@ -469,17 +473,17 @@ public class JsonReaderTestCase { } @Test - public final void testMap() { + public final void testMap() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testmap::whee\"," + " \"fields\": { \"actualmap\": [" + " { \"key\": \"nalle\", \"value\": \"kalle\"}," + " { \"key\": \"tralle\", \"value\": \"skalle\"} ]}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("actualmap")); assertSame(MapFieldValue.class, f.getClass()); @@ -490,15 +494,15 @@ public class JsonReaderTestCase { } @Test - public final void testPositionPositive() { + public final void testPositionPositive() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\"," + " \"fields\": { \"singlepos\": \"N63.429722;E10.393333\" }}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("singlepos")); assertSame(Struct.class, f.getClass()); @@ -507,15 +511,15 @@ public class JsonReaderTestCase { } @Test - public final void testPositionNegative() { + public final void testPositionNegative() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testsinglepos::bamf\"," + " \"fields\": { \"singlepos\": \"W46.63;S23.55\" }}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("singlepos")); assertSame(Struct.class, f.getClass()); @@ -524,7 +528,7 @@ public class JsonReaderTestCase { } @Test - public final void testRaw() { + public final void testRaw() throws IOException { String stuff = new String(new JsonStringEncoder().quoteAsString(new Base64().encodeToString(Utf8.toBytes("smoketest")))); InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testraw::whee\"," @@ -533,10 +537,10 @@ public class JsonReaderTestCase { + "\"" + " }}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("actualraw")); assertSame(Raw.class, f.getClass()); @@ -546,17 +550,17 @@ public class JsonReaderTestCase { } @Test - public final void testMapStringToArrayOfInt() { + public final void testMapStringToArrayOfInt() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:testMapStringToArrayOfInt::whee\"," + " \"fields\": { \"actualMapStringToArrayOfInt\": [" + "{ \"key\": \"bamse\", \"value\": [1, 2, 3] }" + "]}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); Document doc = put.getDocument(); FieldValue f = doc.getFieldValue("actualMapStringToArrayOfInt"); assertSame(MapFieldValue.class, f.getClass()); @@ -569,16 +573,16 @@ public class JsonReaderTestCase { } @Test - public final void testAssignToString() { + public final void testAssignToString() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"update\": \"id:unittest:smoke::whee\"," + " \"fields\": { \"something\": {" + " \"assign\": \"orOther\" }}" + " }")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId); - r.readUpdate(doc); + r.readUpdate(parseInfo.fieldsBuffer, doc); FieldUpdate f = doc.getFieldUpdate("something"); assertEquals(1, f.size()); AssignValueUpdate a = (AssignValueUpdate) f.getValueUpdate(0); @@ -586,7 +590,7 @@ public class JsonReaderTestCase { } @Test - public final void testAssignToArray() { + public final void testAssignToArray() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"update\": \"id:unittest:testMapStringToArrayOfInt::whee\"," + " \"fields\": { \"actualMapStringToArrayOfInt\": {" @@ -594,10 +598,10 @@ public class JsonReaderTestCase { + "{ \"key\": \"bamse\", \"value\": [1, 2, 3] }" + "]}}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId); - r.readUpdate(doc); + r.readUpdate(parseInfo.fieldsBuffer, doc); FieldUpdate f = doc.getFieldUpdate("actualMapStringToArrayOfInt"); assertEquals(1, f.size()); AssignValueUpdate assign = (AssignValueUpdate) f.getValueUpdate(0); @@ -610,7 +614,7 @@ public class JsonReaderTestCase { } @Test - public final void testAssignToWeightedSet() { + public final void testAssignToWeightedSet() throws IOException { InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"update\": \"id:unittest:testset::whee\"," + " \"fields\": { " + "\"actualset\": {" @@ -618,10 +622,10 @@ public class JsonReaderTestCase { + " \"person\": 37," + " \"another person\": 41}}}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentUpdate doc = new DocumentUpdate(docType, parseInfo.documentId); - r.readUpdate(doc); + r.readUpdate(parseInfo.fieldsBuffer, doc); FieldUpdate x = doc.getFieldUpdate("actualset"); assertEquals(1, x.size()); AssignValueUpdate assign = (AssignValueUpdate) x.getValueUpdate(0); @@ -813,18 +817,18 @@ public class JsonReaderTestCase { } @Test - public final void misspelledFieldTest() { + public final void misspelledFieldTest() throws IOException{ InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"put\": \"id:unittest:smoke::whee\"," + " \"fields\": { \"smething\": \"smoketest\"," + " \"nalle\": \"bamse\"}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); exception.expect(NullPointerException.class); exception.expectMessage("Could not get field \"smething\" in the structure of type \"smoke\"."); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); } @Test @@ -842,16 +846,16 @@ public class JsonReaderTestCase { } @Test - public final void idAsAliasForPutTest() { + public final void idAsAliasForPutTest() throws IOException{ InputStream rawDoc = new ByteArrayInputStream( Utf8.toBytes("{\"id\": \"id:unittest:smoke::whee\"," + " \"fields\": { \"something\": \"smoketest\"," + " \"nalle\": \"bamse\"}}")); JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentParseInfo parseInfo = r.parseDocument().get(); DocumentType docType = r.readDocumentType(parseInfo.documentId); DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - r.readPut(put); + r.readPut(parseInfo.fieldsBuffer, put); smokeTestDoc(put.getDocument()); } diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index 89a190beaea..76d18e71a51 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -22,6 +22,7 @@ import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.ReferenceFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.json.readers.DocumentParseInfo; import com.yahoo.tensor.TensorType; import com.yahoo.text.Utf8; import org.apache.commons.codec.binary.Base64; @@ -36,6 +37,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static com.yahoo.document.json.readers.MapReader.MAP_KEY; +import static com.yahoo.document.json.readers.MapReader.MAP_VALUE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; @@ -244,8 +247,8 @@ public class JsonWriterTestCase { private Map<Object, Object> populateMap(List<?> actualMap) { Map<Object, Object> m = new HashMap<>(); for (Object o : actualMap) { - Object key = ((Map) o).get(JsonReader.MAP_KEY); - Object value = ((Map) o).get(JsonReader.MAP_VALUE); + Object key = ((Map) o).get(MAP_KEY); + Object value = ((Map) o).get(MAP_VALUE); m.put(key, value); } return m; @@ -288,13 +291,15 @@ public class JsonWriterTestCase { assertEquals(populateMap(inputMap), populateMap(generatedMap)); } - private Document readDocumentFromJson(String docId, String fields) { + private Document readDocumentFromJson(String docId, String fields) throws IOException { InputStream rawDoc = new ByteArrayInputStream(asFeed(docId, fields)); + + JsonReader r = new JsonReader(types, rawDoc, parserFactory); - JsonReader.DocumentParseInfo raw = r.parseDocument().get(); + DocumentParseInfo raw = r.parseDocument().get(); DocumentType docType = r.readDocumentType(raw.documentId); DocumentPut put = new DocumentPut(new Document(docType, raw.documentId)); - r.readPut(put); + r.readPut(raw.fieldsBuffer, put); return put.getDocument(); } @@ -388,7 +393,7 @@ public class JsonWriterTestCase { } @Test - public void non_empty_reference_field_results_in_reference_value_with_doc_id_present() { + public void non_empty_reference_field_results_in_reference_value_with_doc_id_present() throws IOException { final Document doc = readDocumentFromJson("id:unittest:testrefs::helloworld", "{ \"ref_field\": \"id:unittest:smoke::and_mirrors_too\" }"); ReferenceFieldValue ref = (ReferenceFieldValue)doc.getFieldValue("ref_field"); @@ -403,7 +408,7 @@ public class JsonWriterTestCase { } @Test - public void empty_reference_field_results_in_reference_value_without_doc_id_present() { + public void empty_reference_field_results_in_reference_value_without_doc_id_present() throws IOException { final Document doc = readDocumentFromJson("id:unittest:testrefs::helloworld", "{ \"ref_field\": \"\" }"); ReferenceFieldValue ref = (ReferenceFieldValue)doc.getFieldValue("ref_field"); assertFalse(ref.getDocumentId().isPresent()); |