diff options
Diffstat (limited to 'document/src')
9 files changed, 1171 insertions, 303 deletions
diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java new file mode 100644 index 00000000000..a059df01ad5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -0,0 +1,341 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.serialization.DocumentUpdateWriter; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.ArithmeticValueUpdate; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.ClearValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.RemoveValueUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.vespa.objects.FieldBase; +import com.yahoo.vespa.objects.Serializer; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; + +import static com.yahoo.document.json.JsonSerializationHelper.*; + +/** + * The DocumentUpdateJsonSerializer utility class is used to serialize a DocumentUpdate instance using the JSON format described in + * <a href="https://git.corp.yahoo.com/pages/vespa/documentation/documentation/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> + * + * @see #serialize(com.yahoo.document.DocumentUpdate) + * @author Vegard Sjonfjell + */ +public class DocumentUpdateJsonSerializer +{ + private final JsonFactory jsonFactory = new JsonFactory(); + private final JsonDocumentUpdateWriter writer = new JsonDocumentUpdateWriter(); + private JsonGenerator generator; + + /** + * Instantiate a DocumentUpdateJsonSerializer that outputs JSON to an OutputStream + */ + public DocumentUpdateJsonSerializer(OutputStream outputStream) { + wrapIOException(() -> generator = jsonFactory.createGenerator(outputStream)); + } + + /** + * Instantiate a DocumentUpdateJsonSerializer that writes JSON using existing JsonGenerator + */ + public DocumentUpdateJsonSerializer(JsonGenerator generator) { + this.generator = generator; + } + + /** + * Serialize a DocumentUpdate tree to JSON + */ + public void serialize(DocumentUpdate update) { + writer.write(update); + } + + private class JsonDocumentUpdateWriter implements DocumentUpdateWriter, FieldWriter { + @Override + public void write(DocumentUpdate update) { + wrapIOException(() -> { + generator.writeStartObject(); + generator.writeStringField("update", update.getId().toString()); + + if (update.getCondition().isPresent()) { + generator.writeStringField("condition", update.getCondition().getSelection()); + } + + generator.writeObjectFieldStart("fields"); + for (FieldUpdate up : update.getFieldUpdates()) { + up.serialize(this); + } + generator.writeEndObject(); + + generator.writeEndObject(); + generator.flush(); + }); + } + + @Override + public void write(FieldUpdate fieldUpdate) { + wrapIOException(() -> { + generator.writeObjectFieldStart(fieldUpdate.getField().getName()); + + ArrayList<ValueUpdate> removeValueUpdates = new ArrayList<>(); + ArrayList<ValueUpdate> addValueUpdates = new ArrayList<>(); + + final DataType dataType = fieldUpdate.getField().getDataType(); + for (ValueUpdate valueUpdate : fieldUpdate.getValueUpdates()) { + if (valueUpdate instanceof RemoveValueUpdate) { + removeValueUpdates.add(valueUpdate); + } else if (valueUpdate instanceof AddValueUpdate) { + addValueUpdates.add(valueUpdate); + } else { + valueUpdate.serialize(this, dataType); + } + } + + writeAddOrRemoveValueUpdates("remove", removeValueUpdates, dataType); + writeAddOrRemoveValueUpdates("add", addValueUpdates, dataType); + + generator.writeEndObject(); + }); + } + + private void writeAddOrRemoveValueUpdates(String arrayFieldName, ArrayList<ValueUpdate> valueUpdates, DataType dataType) throws IOException { + if (!valueUpdates.isEmpty()) { + generator.writeArrayFieldStart(arrayFieldName); + for (ValueUpdate valueUpdate : valueUpdates) { + valueUpdate.serialize(this, dataType); + } + generator.writeEndArray(); + } + } + + @Override + public void write(AddValueUpdate update, DataType superType) { + update.getValue().serialize(this); + } + + /* This is the 'match' operation */ + @Override + public void write(MapValueUpdate update, DataType superType) { + wrapIOException(() -> { + generator.writeObjectFieldStart("match"); + generator.writeFieldName("element"); + update.getValue().serialize(null, this); + update.getUpdate().serialize(this, superType); + generator.writeEndObject(); + }); + } + + @Override + public void write(ArithmeticValueUpdate update) { + final ArithmeticValueUpdate.Operator operator = update.getOperator(); + final String operationKey; + + switch (operator) { + case ADD: + operationKey = "increment"; + break; + case DIV: + operationKey = "divide"; + break; + case MUL: + operationKey = "multiply"; + break; + case SUB: + operationKey = "decrement"; + break; + default: + throw new RuntimeException(String.format("Unrecognized arithmetic operator '%s'", operator.name)); + } + + wrapIOException(() -> generator.writeFieldName(operationKey)); + update.getValue().serialize(this); + } + + @Override + public void write(AssignValueUpdate update, DataType superType) { + wrapIOException(() -> generator.writeFieldName("assign")); + update.getValue().serialize(null, this); + } + + @Override + public void write(RemoveValueUpdate update, DataType superType) { + update.getValue().serialize(null, this); + } + + @Override + public void write(ClearValueUpdate clearValueUpdate, DataType superType) { + wrapIOException(() -> generator.writeNullField("assign")); + } + + @Override + public void write(FieldBase field, FieldValue value) { + throw new JsonSerializationException(String.format("Serialization of field values of type %s is not supported", value.getClass().getName())); + } + + @Override + public void write(FieldBase field, Document value) { + throw new JsonSerializationException("Serialization of 'Document fields' is not supported"); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, Array<T> array) { + serializeArrayField(this, generator, field, array); + } + + @Override + public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) { + serializeMapField(this, generator, field, map); + } + + @Override + public void write(FieldBase field, ByteFieldValue value) { + serializeByteField(generator, field, value); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { + serializeCollectionField(this, generator, field, value); + } + + @Override + public void write(FieldBase field, DoubleFieldValue value) { + serializeDoubleField(generator, field, value); + } + + @Override + public void write(FieldBase field, FloatFieldValue value) { + serializeFloatField(generator, field, value); + } + + @Override + public void write(FieldBase field, IntegerFieldValue value) { + serializeIntField(generator, field, value); + } + + @Override + public void write(FieldBase field, LongFieldValue value) { + serializeLongField(generator, field, value); + } + + @Override + public void write(FieldBase field, Raw value) { + serializeRawField(generator, field, value); + } + + @Override + public void write(FieldBase field, PredicateFieldValue value) { + serializePredicateField(generator, field, value); + } + + @Override + public void write(FieldBase field, StringFieldValue value) { + serializeStringField(generator, field, value); + } + + @Override + public void write(FieldBase field, TensorFieldValue value) { + serializeTensorField(generator, field, value); + } + + @Override + public void write(FieldBase field, Struct value) { + serializeStructField(this, generator, field, value); + } + + @Override + public void write(FieldBase field, StructuredFieldValue value) { + serializeStructuredField(this, generator, field, value); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> weightedSet) { + serializeWeightedSet(generator, field, weightedSet); + } + + @Override + public void write(FieldBase field, AnnotationReference value) { + // Serialization of annotations are not implemented + } + + @Override + public Serializer putByte(FieldBase field, byte value) { + serializeByte(generator, field, value); + return this; + } + + @Override + public Serializer putShort(FieldBase field, short value) { + serializeShort(generator, field, value); + return this; + } + + @Override + public Serializer putInt(FieldBase field, int value) { + serializeInt(generator, field, value); + return this; + } + + @Override + public Serializer putLong(FieldBase field, long value) { + serializeLong(generator, field, value); + return this; + } + + @Override + public Serializer putFloat(FieldBase field, float value) { + serializeFloat(generator, field, value); + return this; + } + + @Override + public Serializer putDouble(FieldBase field, double value) { + serializeDouble(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, byte[] value) { + serializeByteArray(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, ByteBuffer value) { + serializeByteBuffer(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, String value) { + serializeString(generator, field, value); + return this; + } + } +}
\ No newline at end of file diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java new file mode 100644 index 00000000000..639c6ad5300 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -0,0 +1,297 @@ +package com.yahoo.document.json; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.document.Field; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorAddress; +import com.yahoo.vespa.objects.FieldBase; +import org.apache.commons.codec.binary.Base64; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * @author Steinar Knutsen + * @author Vegard Sjonfjell + */ +public class JsonSerializationHelper { + private final static Base64 base64Encoder = new Base64(); + + static class JsonSerializationException extends RuntimeException { + public JsonSerializationException(Exception base) { + super(base); + } + + public JsonSerializationException(String message) { + super(message); + } + } + + @FunctionalInterface + static interface SubroutineThrowingIOException { + void invoke() throws IOException; + } + + static void wrapIOException(SubroutineThrowingIOException lambda) { + try { + lambda.invoke(); + } catch (IOException e) { + throw new JsonSerializationException(e); + } + } + + public static void serializeTensorField(JsonGenerator generator, FieldBase field, TensorFieldValue value) { + wrapIOException(() -> { + fieldNameIfNotNull(generator, field); + generator.writeStartObject(); + + if (value.getTensor().isPresent()) { + Tensor tensor = value.getTensor().get(); + serializeTensorDimensions(generator, tensor.dimensions()); + serializeTensorCells(generator, tensor.cells()); + } + generator.writeEndObject(); + }); + } + + private static void serializeTensorDimensions(JsonGenerator generator, Set<String> dimensions) throws IOException { + generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS); + for (String dimension : dimensions) { + generator.writeString(dimension); + } + + generator.writeEndArray(); + } + + private static void serializeTensorCells(JsonGenerator generator, Map<TensorAddress, Double> cells) throws IOException { + generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS); + for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) { + generator.writeStartObject(); + serializeTensorAddress(generator, cell.getKey()); + generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue()); + generator.writeEndObject(); + } + + generator.writeEndArray(); + } + + private static void serializeTensorAddress(JsonGenerator generator, TensorAddress address) throws IOException { + generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS); + for (TensorAddress.Element element : address.elements()) { + generator.writeStringField(element.dimension(), element.label()); + } + + generator.writeEndObject(); + } + + + public static void serializeStringField(JsonGenerator generator, FieldBase field, StringFieldValue value) { + // Hide empty strings + if (value.getString().length() == 0) { + return; + } + + serializeString(generator, field, value.getString()); + } + + public static void serializeStructuredField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, StructuredFieldValue value) { + fieldNameIfNotNull(generator, field); + + wrapIOException(() -> { + generator.writeStartObject(); + Iterator<Map.Entry<Field, FieldValue>> i = value.iterator(); + + while (i.hasNext()) { + Map.Entry<Field, FieldValue> entry = i.next(); + entry.getValue().serialize(entry.getKey(), fieldWriter); + } + + generator.writeEndObject(); + }); + } + + public static void serializeStructField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Struct value) { + if (value.getDataType() == PositionDataType.INSTANCE) { + serializeString(generator, field, PositionDataType.renderAsString(value)); + return; + } + + serializeStructuredField(fieldWriter, generator, field, value); + } + + public static <T extends FieldValue> void serializeWeightedSet(JsonGenerator generator, FieldBase field, WeightedSet<T> value) { + fieldNameIfNotNull(generator, field); + + wrapIOException(() -> { + generator.writeStartObject(); + + for (T key : value.keySet()) { + Integer weight = value.get(key); + // key.toString() is according to spec + generator.writeNumberField(key.toString(), weight); + } + + generator.writeEndObject(); + }); + } + + public static <T extends FieldValue> void serializeCollectionField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, CollectionFieldValue<T> value) { + fieldNameIfNotNull(generator, field); + + wrapIOException(() -> { + generator.writeStartArray(); + Iterator<T> i = value.iterator(); + + while (i.hasNext()) { + i.next().serialize(null, fieldWriter); + } + + generator.writeEndArray(); + }); + } + + + public static <K extends FieldValue, V extends FieldValue> void serializeMapField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, MapFieldValue<K, V> map) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> { + generator.writeStartArray(); + + for (Map.Entry<K, V> entry : map.entrySet()) { + generator.writeStartObject(); + generator.writeFieldName(JsonReader.MAP_KEY); + entry.getKey().serialize(null, fieldWriter); + generator.writeFieldName(JsonReader.MAP_VALUE); + entry.getValue().serialize(null, fieldWriter); + generator.writeEndObject(); + } + + generator.writeEndArray(); + }); + } + + public static <T extends FieldValue> void serializeArrayField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Array<T> value) { + wrapIOException(() -> { + fieldNameIfNotNull(generator, field); + generator.writeStartArray(); + + for (T elem : value) { + elem.serialize(null, fieldWriter); + } + + generator.writeEndArray(); + }); + } + + public static void serializeDoubleField(JsonGenerator generator, FieldBase field, DoubleFieldValue value) { + serializeDouble(generator, field, value.getDouble()); + } + + public static void serializeFloatField(JsonGenerator generator, FieldBase field, FloatFieldValue value) { + serializeFloat(generator, field, value.getFloat()); + } + + public static void serializeIntField(JsonGenerator generator, FieldBase field, IntegerFieldValue value) { + serializeInt(generator, field, value.getInteger()); + } + + public static void serializeLongField(JsonGenerator generator, FieldBase field, LongFieldValue value) { + serializeLong(generator, field, value.getLong()); + } + + public static void serializeByteField(JsonGenerator generator, FieldBase field, ByteFieldValue value) { + serializeByte(generator, field, value.getByte()); + } + + public static void serializePredicateField(JsonGenerator generator, FieldBase field, PredicateFieldValue value){ + serializeString(generator, field, value.toString()); + } + + public static void serializeRawField(JsonGenerator generator, FieldBase field, Raw raw) { + serializeByteBuffer(generator, field, raw.getByteBuffer()); + } + + public static void serializeString(JsonGenerator generator, FieldBase field, String value) { + if (value.length() == 0) { + return; + } + + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeString(value)); + } + + public static void serializeByte(JsonGenerator generator, FieldBase field, byte value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeShort(JsonGenerator generator, FieldBase field, short value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeInt(JsonGenerator generator, FieldBase field, int value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeLong(JsonGenerator generator, FieldBase field, long value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeFloat(JsonGenerator generator, FieldBase field, float value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeDouble(JsonGenerator generator, FieldBase field, double value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeByteBuffer(JsonGenerator generator, FieldBase field, ByteBuffer raw) { + fieldNameIfNotNull(generator, field); + + final byte[] data = new byte[raw.remaining()]; + final int origPosition = raw.position(); + + // base64encoder has no encode methods with offset and + // limit, so no use trying to get at the backing array if + // available anyway + raw.get(data); + raw.position(origPosition); + + wrapIOException(() -> generator.writeString(base64Encoder.encodeToString(data))); + } + + public static void serializeByteArray(JsonGenerator generator, FieldBase field, byte[] value) { + serializeByteBuffer(generator, field, ByteBuffer.wrap(value)); + } + + public static void fieldNameIfNotNull(JsonGenerator generator, FieldBase field) { + if (field != null) { + wrapIOException(() -> generator.writeFieldName(field.getName())); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index 79a1c040cbc..626c97a958a 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -1,34 +1,43 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.json; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import com.yahoo.document.datatypes.*; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorAddress; -import org.apache.commons.codec.binary.Base64; - import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; -import com.yahoo.document.PositionDataType; import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.document.serialization.DocumentWriter; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.vespa.objects.Serializer; - import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.Map; + +import static com.yahoo.document.json.JsonSerializationHelper.*; + /** * Serialize Document and other FieldValue instances as JSON. * @@ -38,7 +47,6 @@ public class JsonWriter implements DocumentWriter { private static final JsonFactory jsonFactory = new JsonFactory(); private final JsonGenerator generator; - private final Base64 base64Encoder = new Base64(); // I really hate exception unsafe constructors, but the alternative // requires generator to not be a final @@ -85,22 +93,23 @@ public class JsonWriter implements DocumentWriter { */ @Override public void write(FieldBase field, FieldValue value) { - throw new UnsupportedOperationException("Serializing " - + value.getClass().getName() + " is not supported."); + throw new UnsupportedOperationException("Serializing " + value.getClass().getName() + " is not supported."); } @Override public void write(FieldBase field, Document value) { try { - fieldNameIfNotNull(field); + fieldNameIfNotNull(generator, field); generator.writeStartObject(); + // this makes it impossible to refeed directly, not sure what's correct // perhaps just change to "put"? generator.writeStringField("id", value.getId().toString()); generator.writeObjectFieldStart(JsonReader.FIELDS); - for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i - .hasNext();) { - Entry<Field, FieldValue> entry = i.next(); + + Iterator<Map.Entry<Field, FieldValue>> i = value.iterator(); + while (i.hasNext()) { + Map.Entry<Field, FieldValue> entry = i.next(); entry.getValue().serialize(entry.getKey(), this); } generator.writeEndObject(); @@ -113,199 +122,77 @@ public class JsonWriter implements DocumentWriter { @Override public <T extends FieldValue> void write(FieldBase field, Array<T> value) { - try { - fieldNameIfNotNull(field); - generator.writeStartArray(); - for (Iterator<T> i = value.iterator(); i.hasNext();) { - i.next().serialize(null, this); - } - generator.writeEndArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - } - - private void fieldNameIfNotNull(FieldBase field) { - if (field != null) { - try { - generator.writeFieldName(field.getName()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + serializeArrayField(this, generator, field, value); } @Override - public <K extends FieldValue, V extends FieldValue> void write( - FieldBase field, MapFieldValue<K, V> map) { - fieldNameIfNotNull(field); - try { - generator.writeStartArray(); - for (Map.Entry<K, V> entry : map.entrySet()) { - generator.writeStartObject(); - generator.writeFieldName(JsonReader.MAP_KEY); - entry.getKey().serialize(null, this); - generator.writeFieldName(JsonReader.MAP_VALUE); - entry.getValue().serialize(null, this); - generator.writeEndObject(); - } - generator.writeEndArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } + public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) { + serializeMapField(this, generator, field, map); } @Override public void write(FieldBase field, ByteFieldValue value) { - putByte(field, value.getByte()); + serializeByteField(generator, field, value); } @Override - public <T extends FieldValue> void write(FieldBase field, - CollectionFieldValue<T> value) { - fieldNameIfNotNull(field); - try { - generator.writeStartArray(); - for (Iterator<T> i = value.iterator(); i.hasNext();) { - i.next().serialize(null, this); - } - generator.writeEndArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } + public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { + serializeCollectionField(this, generator, field, value); } @Override public void write(FieldBase field, DoubleFieldValue value) { - putDouble(field, value.getDouble()); + serializeDoubleField(generator, field, value); } @Override public void write(FieldBase field, FloatFieldValue value) { - putFloat(field, value.getFloat()); + serializeFloatField(generator, field, value); } @Override public void write(FieldBase field, IntegerFieldValue value) { - putInt(field, value.getInteger()); + serializeIntField(generator, field, value); } @Override public void write(FieldBase field, LongFieldValue value) { - putLong(field, value.getLong()); + serializeLongField(generator, field, value); } @Override public void write(FieldBase field, Raw value) { - put(field, value.getByteBuffer()); + serializeRawField(generator, field, value); } @Override public void write(FieldBase field, PredicateFieldValue value) { - put(field, value.toString()); + serializePredicateField(generator, field, value); } @Override public void write(FieldBase field, StringFieldValue value) { - put(field, value.getString()); + serializeStringField(generator, field, value); } @Override public void write(FieldBase field, TensorFieldValue value) { - try { - fieldNameIfNotNull(field); - generator.writeStartObject(); - if (value.getTensor().isPresent()) { - Tensor tensor = value.getTensor().get(); - writeTensorDimensions(tensor.dimensions()); - writeTensorCells(tensor.cells()); - } - generator.writeEndObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private void writeTensorDimensions(Set<String> dimensions) throws IOException { - generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS); - for (String dimension : dimensions) { - generator.writeString(dimension); - } - generator.writeEndArray(); - } - - private void writeTensorCells(Map<TensorAddress, Double> cells) throws IOException { - generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS); - for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) { - generator.writeStartObject(); - writeTensorAddress(cell.getKey()); - generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue()); - generator.writeEndObject(); - } - generator.writeEndArray(); - } - - private void writeTensorAddress(TensorAddress address) throws IOException { - generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS); - for (TensorAddress.Element element : address.elements()) { - generator.writeStringField(element.dimension(), element.label()); - } - generator.writeEndObject(); + serializeTensorField(generator, field, value); } @Override public void write(FieldBase field, Struct value) { - if (value.getDataType() == PositionDataType.INSTANCE) { - put(field, PositionDataType.renderAsString(value)); - return; - } - fieldNameIfNotNull(field); - try { - generator.writeStartObject(); - for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i - .hasNext();) { - Entry<Field, FieldValue> entry = i.next(); - entry.getValue().serialize(entry.getKey(), this); - } - generator.writeEndObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } + serializeStructField(this, generator, field, value); } @Override public void write(FieldBase field, StructuredFieldValue value) { - fieldNameIfNotNull(field); - try { - generator.writeStartObject(); - for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i - .hasNext();) { - Entry<Field, FieldValue> entry = i.next(); - entry.getValue().serialize(entry.getKey(), this); - } - generator.writeEndObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } + serializeStructuredField(this, generator, field, value); } @Override - public <T extends FieldValue> void write(FieldBase field, - WeightedSet<T> value) { - fieldNameIfNotNull(field); - try { - generator.writeStartObject(); - // entrySet() is deprecated and there is no entry iterator - for (T key : value.keySet()) { - Integer weight = value.get(key); - // key.toString() is according to spec - generator.writeNumberField(key.toString(), weight); - } - generator.writeEndObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } + public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> value) { + serializeWeightedSet(generator, field, value); } @Override @@ -315,110 +202,6 @@ public class JsonWriter implements DocumentWriter { } @Override - public Serializer putByte(FieldBase field, byte value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putShort(FieldBase field, short value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putInt(FieldBase field, int value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putLong(FieldBase field, long value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putFloat(FieldBase field, float value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putDouble(FieldBase field, double value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer put(FieldBase field, byte[] value) { - return put(field, ByteBuffer.wrap(value)); - } - - @Override - public Serializer put(FieldBase field, ByteBuffer raw) { - final byte[] data = new byte[raw.remaining()]; - final int origPosition = raw.position(); - - fieldNameIfNotNull(field); - // base64encoder has no encode methods with offset and - // limit, so no use trying to get at the backing array if - // available anyway - raw.get(data); - raw.position(origPosition); - try { - generator.writeString(base64Encoder.encodeToString(data)); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer put(FieldBase field, String value) { - if (value.length() == 0) { - return this; - } - fieldNameIfNotNull(field); - try { - generator.writeString(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override public void write(Document document) { write(null, document); } @@ -470,4 +253,58 @@ public class JsonWriter implements DocumentWriter { } return out.toByteArray(); } + + @Override + public Serializer putByte(FieldBase field, byte value) { + serializeByte(generator, field, value); + return this; + } + + @Override + public Serializer putShort(FieldBase field, short value) { + serializeShort(generator, field, value); + return this; + } + + @Override + public Serializer putInt(FieldBase field, int value) { + serializeInt(generator, field, value); + return this; + } + + @Override + public Serializer putLong(FieldBase field, long value) { + serializeLong(generator, field, value); + return this; + } + + @Override + public Serializer putFloat(FieldBase field, float value) { + serializeFloat(generator, field, value); + return this; + } + + @Override + public Serializer putDouble(FieldBase field, double value) { + serializeDouble(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, byte[] value) { + serializeByteArray(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, ByteBuffer value) { + serializeByteBuffer(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, String value) { + serializeString(generator, field, value); + return this; + } } diff --git a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java index 4a269a704d2..244aa0d5a0d 100644 --- a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java @@ -3,7 +3,22 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; import com.yahoo.document.annotation.AnnotationReference; -import com.yahoo.document.datatypes.*; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.vespa.objects.Serializer; @@ -189,5 +204,4 @@ public interface FieldWriter extends Serializer { * field value */ void write(FieldBase field, AnnotationReference value); - } diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java index ffe6073cd6c..dc605fbe3d5 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -1,21 +1,36 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.Iterator; -import java.util.Map; - import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.annotation.AnnotationReference; -import com.yahoo.document.datatypes.*; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.vespa.objects.Serializer; +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.Map; + // TODO: Just inline all use of XmlSerializationHelper when the toXml methods in FieldValue subclasses are to be removed // TODO: More cleanup, the put() methods generate a lot of superfluous objects (write should call put, not the other way around) // TODO: remove pingpong between XmlSerializationHelper and FieldValue, this will go away when the toXml methods go away diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java new file mode 100644 index 00000000000..53e7ed701a7 --- /dev/null +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -0,0 +1,342 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import com.fasterxml.jackson.core.JsonFactory; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.text.Utf8; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; + +import static com.yahoo.document.json.JsonTestHelper.assertJsonEquals; +import static com.yahoo.document.json.JsonTestHelper.inputJson; + +/** + * @author Vegard Sjonfjell + */ +public class DocumentUpdateJsonSerializerTest { + final static DocumentTypeManager types = new DocumentTypeManager(); + final static JsonFactory parserFactory = new JsonFactory(); + final static DocumentType docType = new DocumentType("doctype"); + + final static String DEFAULT_DOCUMENT_ID = "id:test:doctype::1"; + + static { + docType.addField(new Field("string_field", DataType.STRING)); + docType.addField(new Field("int_field", DataType.INT)); + docType.addField(new Field("float_field", DataType.FLOAT)); + docType.addField(new Field("double_field", DataType.DOUBLE)); + docType.addField(new Field("byte_field", DataType.BYTE)); + docType.addField(new Field("tensor_field", DataType.TENSOR)); + docType.addField(new Field("predicate_field", DataType.PREDICATE)); + docType.addField(new Field("raw_field", DataType.RAW)); + docType.addField(new Field("int_array", new ArrayDataType(DataType.INT))); + docType.addField(new Field("string_array", new ArrayDataType(DataType.STRING))); + docType.addField(new Field("int_set", new WeightedSetDataType(DataType.INT, true, true))); + docType.addField(new Field("string_set", new WeightedSetDataType(DataType.STRING, true, true))); + docType.addField(new Field("string_map", new MapDataType(DataType.STRING, DataType.STRING))); + docType.addField(new Field("singlepos_field", PositionDataType.INSTANCE)); + docType.addField(new Field("multipos_field", new ArrayDataType(PositionDataType.INSTANCE))); + types.registerDocumentType(docType); + } + + private static DocumentUpdate deSerializeDocumentUpdate(String jsonDoc, String docId) { + final InputStream rawDoc = new ByteArrayInputStream(Utf8.toBytes(jsonDoc)); + JsonReader reader = new JsonReader(types, rawDoc, parserFactory); + return (DocumentUpdate) reader.readSingleDocument(JsonReader.SupportedOperation.UPDATE, docId); + } + + private static String serializeDocumentUpdate(DocumentUpdate update) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + DocumentUpdateJsonSerializer serializer = new DocumentUpdateJsonSerializer(outputStream); + serializer.serialize(update); + + try { + return new String(outputStream.toByteArray(), "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + private static void deSerializeAndSerializeJsonAndMatch(String jsonDoc) { + jsonDoc = jsonDoc.replaceFirst("DOCUMENT_ID", DEFAULT_DOCUMENT_ID); + DocumentUpdate update = deSerializeDocumentUpdate(jsonDoc, DEFAULT_DOCUMENT_ID); + assertJsonEquals(serializeDocumentUpdate(update), jsonDoc); + } + + @Test + public void testArithmeticUpdate() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'int_field': {", + " 'increment': 3.0", + " },", + " 'float_field': {", + " 'decrement': 1.5", + " },", + " 'double_field': {", + " 'divide': 3.2", + " },", + " 'byte_field': {", + " 'multiply': 2.0", + " }", + " }", + "}" + )); + } + + @Test + public void testAssignSimpleTypes() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'int_field': {", + " 'assign': 42", + " },", + " 'float_field': {", + " 'assign': 32.45", + " },", + " 'double_field': {", + " 'assign': 45.93", + " },", + " 'string_field': {", + " 'assign': \"My favorite string\"", + " },", + " 'byte_field': {", + " 'assign': 127", + " }", + " }", + "}" + )); + } + + @Test + public void testAssignWeightedSet() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'int_set': {", + " 'assign': {", + " '123': 456,", + " '789': 101112", + " }", + " },", + " 'string_set': {", + " 'assign': {", + " 'meow': 218478,", + " 'slurp': 2123", + " }", + " }", + " }", + "}" + )); + } + + @Test + public void testAddUpdate() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'int_array': {", + " 'add': [", + " 123,", + " 456,", + " 789", + " ]", + " },", + " 'string_array': {", + " 'add': [", + " 'bjarne',", + " 'andrei',", + " 'rich'", + " ]", + " }", + " }", + "}" + )); + } + + @Test + public void testRemoveUpdate() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'int_array': {", + " 'remove': [", + " 123,", + " 789", + " ]", + " },", + " 'string_array': {", + " 'remove': [", + " 'bjarne',", + " 'rich'", + " ]", + " }", + " }", + "}" + )); + } + + @Test + public void testMatchUpdateArithmetic() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'int_array': {", + " 'match': {", + " 'element': 456,", + " 'multiply': 8.0", + " }", + " }", + " }", + "}" + )); + } + + @Test + public void testMatchUpdateAssign() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'string_array': {", + " 'match': {", + " 'element': 3,", + " 'assign': 'kjeks'", + " }", + " }", + " }", + "}" + )); + } + + @Test + public void testAssignTensor() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'tensor_field': {", + " 'assign': {", + " 'dimensions': ['x','y','z'], ", + " 'cells': [", + " { 'address': { 'x': 'a', 'y': 'b' }, 'value': 2.0 },", + " { 'address': { 'x': 'c' }, 'value': 3.0 }", + " ]", + " }", + " }", + " }", + "}" + )); + } + + @Test + public void testAssignPredicate() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'predicate_field': {", + " 'assign': 'foo in [bar]'", + " }", + " }", + "}" + )); + } + + @Test + public void testAssignRaw() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'raw_field': {", + " 'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw==\\r\\n'", + " }", + " }", + "}" + )); + } + + @Test + public void testAssignMap() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'string_map': {", + " 'assign': [", + " { 'key': 'conversion gel', 'value': 'deadly'},", + " { 'key': 'repulsion gel', 'value': 'safe'},", + " { 'key': 'propulsion gel', 'value': 'insufficient data'}", + " ]", + " }", + " }", + "}" + )); + } + + @Test + public void testAssignSinglePos() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'singlepos_field': {", + " 'assign': 'N60.222333;E10.12'", + " }", + " }", + "}" + )); + } + + @Test + public void testAssignMultiPos() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'multipos_field': {", + " 'assign': [ 'N0.0;E0.0', 'S1.1;W1.1', 'N10.2;W122.2' ]", + " }", + " }", + "}" + )); + } + + @Test + public void testClearField() { + deSerializeAndSerializeJsonAndMatch(inputJson( + "{", + " 'update': 'DOCUMENT_ID',", + " 'fields': {", + " 'int_field': {", + " 'assign': null", + " },", + " 'string_field': {", + " 'assign': null", + " }", + " }", + "}" + )); + } +} diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 8236a32e94c..466a915f83f 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -29,8 +29,14 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.datatypes.WeightedSet; -import com.yahoo.document.update.*; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.ArithmeticValueUpdate; import com.yahoo.document.update.ArithmeticValueUpdate.Operator; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.ClearValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.ValueUpdate; import com.yahoo.tensor.MapTensor; import com.yahoo.text.Utf8; import org.apache.commons.codec.binary.Base64; @@ -53,6 +59,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import static com.yahoo.document.json.JsonTestHelper.inputJson; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.*; @@ -60,7 +67,6 @@ import static org.junit.Assert.*; * Basic test of JSON streams to Vespa document instances. * * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @author vegard */ public class JsonReaderTestCase { DocumentTypeManager types; @@ -1011,15 +1017,6 @@ public class JsonReaderTestCase { return new ByteArrayInputStream(Utf8.toBytes(json)); } - /** - * Convenience method to input JSON without escaping double quotes and newlines - * Each parameter represents a line of JSON encoded data - * The lines are joined with newline and single quotes are replaced with double quotes - */ - static String inputJson(String... lines) { - return Joiner.on("\n").join(lines).replaceAll("'", "\""); - } - @Test public void testParsingWithoutTensorField() { Document doc = createPutWithoutTensor().getDocument(); diff --git a/document/src/test/java/com/yahoo/document/json/JsonTestHelper.java b/document/src/test/java/com/yahoo/document/json/JsonTestHelper.java new file mode 100644 index 00000000000..45128e6d02a --- /dev/null +++ b/document/src/test/java/com/yahoo/document/json/JsonTestHelper.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; +import com.google.common.base.Joiner; +import static org.hamcrest.MatcherAssert.assertThat; +import static uk.co.datumedge.hamcrest.json.SameJSONAs.sameJSONAs; + +/** + * @author Vegard Sjonfjell + */ +public class JsonTestHelper { + /** + * Convenience method to input JSON without escaping double quotes and newlines + * Each parameter represents a line of JSON encoded data + * The lines are joined with newline and single quotes are replaced with double quotes + */ + public static String inputJson(String... lines) { + return Joiner.on("\n").join(lines).replaceAll("'", "\""); + } + + /** + * Structurally compare two JSON encoded strings + */ + public static void assertJsonEquals(String inputJson, String expectedJson) { + assertThat(inputJson, sameJSONAs(expectedJson)); + } +} diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index 2e8354e3c6a..a587499a3c6 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -1,16 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.json; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.io.JsonStringEncoder; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Document; @@ -26,17 +21,21 @@ import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.text.Utf8; import org.apache.commons.codec.binary.Base64; import org.junit.After; import org.junit.Before; import org.junit.Test; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.io.JsonStringEncoder; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.text.Utf8; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; /** * Functional tests for com.yahoo.document.json.JsonWriter. |