diff options
Diffstat (limited to 'document/src/main/java/com/yahoo')
5 files changed, 780 insertions, 276 deletions
diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java new file mode 100644 index 00000000000..a059df01ad5 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -0,0 +1,341 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.serialization.DocumentUpdateWriter; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.ArithmeticValueUpdate; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.ClearValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.RemoveValueUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.vespa.objects.FieldBase; +import com.yahoo.vespa.objects.Serializer; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.ArrayList; + +import static com.yahoo.document.json.JsonSerializationHelper.*; + +/** + * The DocumentUpdateJsonSerializer utility class is used to serialize a DocumentUpdate instance using the JSON format described in + * <a href="https://git.corp.yahoo.com/pages/vespa/documentation/documentation/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> + * + * @see #serialize(com.yahoo.document.DocumentUpdate) + * @author Vegard Sjonfjell + */ +public class DocumentUpdateJsonSerializer +{ + private final JsonFactory jsonFactory = new JsonFactory(); + private final JsonDocumentUpdateWriter writer = new JsonDocumentUpdateWriter(); + private JsonGenerator generator; + + /** + * Instantiate a DocumentUpdateJsonSerializer that outputs JSON to an OutputStream + */ + public DocumentUpdateJsonSerializer(OutputStream outputStream) { + wrapIOException(() -> generator = jsonFactory.createGenerator(outputStream)); + } + + /** + * Instantiate a DocumentUpdateJsonSerializer that writes JSON using existing JsonGenerator + */ + public DocumentUpdateJsonSerializer(JsonGenerator generator) { + this.generator = generator; + } + + /** + * Serialize a DocumentUpdate tree to JSON + */ + public void serialize(DocumentUpdate update) { + writer.write(update); + } + + private class JsonDocumentUpdateWriter implements DocumentUpdateWriter, FieldWriter { + @Override + public void write(DocumentUpdate update) { + wrapIOException(() -> { + generator.writeStartObject(); + generator.writeStringField("update", update.getId().toString()); + + if (update.getCondition().isPresent()) { + generator.writeStringField("condition", update.getCondition().getSelection()); + } + + generator.writeObjectFieldStart("fields"); + for (FieldUpdate up : update.getFieldUpdates()) { + up.serialize(this); + } + generator.writeEndObject(); + + generator.writeEndObject(); + generator.flush(); + }); + } + + @Override + public void write(FieldUpdate fieldUpdate) { + wrapIOException(() -> { + generator.writeObjectFieldStart(fieldUpdate.getField().getName()); + + ArrayList<ValueUpdate> removeValueUpdates = new ArrayList<>(); + ArrayList<ValueUpdate> addValueUpdates = new ArrayList<>(); + + final DataType dataType = fieldUpdate.getField().getDataType(); + for (ValueUpdate valueUpdate : fieldUpdate.getValueUpdates()) { + if (valueUpdate instanceof RemoveValueUpdate) { + removeValueUpdates.add(valueUpdate); + } else if (valueUpdate instanceof AddValueUpdate) { + addValueUpdates.add(valueUpdate); + } else { + valueUpdate.serialize(this, dataType); + } + } + + writeAddOrRemoveValueUpdates("remove", removeValueUpdates, dataType); + writeAddOrRemoveValueUpdates("add", addValueUpdates, dataType); + + generator.writeEndObject(); + }); + } + + private void writeAddOrRemoveValueUpdates(String arrayFieldName, ArrayList<ValueUpdate> valueUpdates, DataType dataType) throws IOException { + if (!valueUpdates.isEmpty()) { + generator.writeArrayFieldStart(arrayFieldName); + for (ValueUpdate valueUpdate : valueUpdates) { + valueUpdate.serialize(this, dataType); + } + generator.writeEndArray(); + } + } + + @Override + public void write(AddValueUpdate update, DataType superType) { + update.getValue().serialize(this); + } + + /* This is the 'match' operation */ + @Override + public void write(MapValueUpdate update, DataType superType) { + wrapIOException(() -> { + generator.writeObjectFieldStart("match"); + generator.writeFieldName("element"); + update.getValue().serialize(null, this); + update.getUpdate().serialize(this, superType); + generator.writeEndObject(); + }); + } + + @Override + public void write(ArithmeticValueUpdate update) { + final ArithmeticValueUpdate.Operator operator = update.getOperator(); + final String operationKey; + + switch (operator) { + case ADD: + operationKey = "increment"; + break; + case DIV: + operationKey = "divide"; + break; + case MUL: + operationKey = "multiply"; + break; + case SUB: + operationKey = "decrement"; + break; + default: + throw new RuntimeException(String.format("Unrecognized arithmetic operator '%s'", operator.name)); + } + + wrapIOException(() -> generator.writeFieldName(operationKey)); + update.getValue().serialize(this); + } + + @Override + public void write(AssignValueUpdate update, DataType superType) { + wrapIOException(() -> generator.writeFieldName("assign")); + update.getValue().serialize(null, this); + } + + @Override + public void write(RemoveValueUpdate update, DataType superType) { + update.getValue().serialize(null, this); + } + + @Override + public void write(ClearValueUpdate clearValueUpdate, DataType superType) { + wrapIOException(() -> generator.writeNullField("assign")); + } + + @Override + public void write(FieldBase field, FieldValue value) { + throw new JsonSerializationException(String.format("Serialization of field values of type %s is not supported", value.getClass().getName())); + } + + @Override + public void write(FieldBase field, Document value) { + throw new JsonSerializationException("Serialization of 'Document fields' is not supported"); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, Array<T> array) { + serializeArrayField(this, generator, field, array); + } + + @Override + public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) { + serializeMapField(this, generator, field, map); + } + + @Override + public void write(FieldBase field, ByteFieldValue value) { + serializeByteField(generator, field, value); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { + serializeCollectionField(this, generator, field, value); + } + + @Override + public void write(FieldBase field, DoubleFieldValue value) { + serializeDoubleField(generator, field, value); + } + + @Override + public void write(FieldBase field, FloatFieldValue value) { + serializeFloatField(generator, field, value); + } + + @Override + public void write(FieldBase field, IntegerFieldValue value) { + serializeIntField(generator, field, value); + } + + @Override + public void write(FieldBase field, LongFieldValue value) { + serializeLongField(generator, field, value); + } + + @Override + public void write(FieldBase field, Raw value) { + serializeRawField(generator, field, value); + } + + @Override + public void write(FieldBase field, PredicateFieldValue value) { + serializePredicateField(generator, field, value); + } + + @Override + public void write(FieldBase field, StringFieldValue value) { + serializeStringField(generator, field, value); + } + + @Override + public void write(FieldBase field, TensorFieldValue value) { + serializeTensorField(generator, field, value); + } + + @Override + public void write(FieldBase field, Struct value) { + serializeStructField(this, generator, field, value); + } + + @Override + public void write(FieldBase field, StructuredFieldValue value) { + serializeStructuredField(this, generator, field, value); + } + + @Override + public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> weightedSet) { + serializeWeightedSet(generator, field, weightedSet); + } + + @Override + public void write(FieldBase field, AnnotationReference value) { + // Serialization of annotations are not implemented + } + + @Override + public Serializer putByte(FieldBase field, byte value) { + serializeByte(generator, field, value); + return this; + } + + @Override + public Serializer putShort(FieldBase field, short value) { + serializeShort(generator, field, value); + return this; + } + + @Override + public Serializer putInt(FieldBase field, int value) { + serializeInt(generator, field, value); + return this; + } + + @Override + public Serializer putLong(FieldBase field, long value) { + serializeLong(generator, field, value); + return this; + } + + @Override + public Serializer putFloat(FieldBase field, float value) { + serializeFloat(generator, field, value); + return this; + } + + @Override + public Serializer putDouble(FieldBase field, double value) { + serializeDouble(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, byte[] value) { + serializeByteArray(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, ByteBuffer value) { + serializeByteBuffer(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, String value) { + serializeString(generator, field, value); + return this; + } + } +}
\ No newline at end of file diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java new file mode 100644 index 00000000000..639c6ad5300 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -0,0 +1,297 @@ +package com.yahoo.document.json; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.yahoo.document.Field; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorAddress; +import com.yahoo.vespa.objects.FieldBase; +import org.apache.commons.codec.binary.Base64; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +/** + * @author Steinar Knutsen + * @author Vegard Sjonfjell + */ +public class JsonSerializationHelper { + private final static Base64 base64Encoder = new Base64(); + + static class JsonSerializationException extends RuntimeException { + public JsonSerializationException(Exception base) { + super(base); + } + + public JsonSerializationException(String message) { + super(message); + } + } + + @FunctionalInterface + static interface SubroutineThrowingIOException { + void invoke() throws IOException; + } + + static void wrapIOException(SubroutineThrowingIOException lambda) { + try { + lambda.invoke(); + } catch (IOException e) { + throw new JsonSerializationException(e); + } + } + + public static void serializeTensorField(JsonGenerator generator, FieldBase field, TensorFieldValue value) { + wrapIOException(() -> { + fieldNameIfNotNull(generator, field); + generator.writeStartObject(); + + if (value.getTensor().isPresent()) { + Tensor tensor = value.getTensor().get(); + serializeTensorDimensions(generator, tensor.dimensions()); + serializeTensorCells(generator, tensor.cells()); + } + generator.writeEndObject(); + }); + } + + private static void serializeTensorDimensions(JsonGenerator generator, Set<String> dimensions) throws IOException { + generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS); + for (String dimension : dimensions) { + generator.writeString(dimension); + } + + generator.writeEndArray(); + } + + private static void serializeTensorCells(JsonGenerator generator, Map<TensorAddress, Double> cells) throws IOException { + generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS); + for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) { + generator.writeStartObject(); + serializeTensorAddress(generator, cell.getKey()); + generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue()); + generator.writeEndObject(); + } + + generator.writeEndArray(); + } + + private static void serializeTensorAddress(JsonGenerator generator, TensorAddress address) throws IOException { + generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS); + for (TensorAddress.Element element : address.elements()) { + generator.writeStringField(element.dimension(), element.label()); + } + + generator.writeEndObject(); + } + + + public static void serializeStringField(JsonGenerator generator, FieldBase field, StringFieldValue value) { + // Hide empty strings + if (value.getString().length() == 0) { + return; + } + + serializeString(generator, field, value.getString()); + } + + public static void serializeStructuredField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, StructuredFieldValue value) { + fieldNameIfNotNull(generator, field); + + wrapIOException(() -> { + generator.writeStartObject(); + Iterator<Map.Entry<Field, FieldValue>> i = value.iterator(); + + while (i.hasNext()) { + Map.Entry<Field, FieldValue> entry = i.next(); + entry.getValue().serialize(entry.getKey(), fieldWriter); + } + + generator.writeEndObject(); + }); + } + + public static void serializeStructField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Struct value) { + if (value.getDataType() == PositionDataType.INSTANCE) { + serializeString(generator, field, PositionDataType.renderAsString(value)); + return; + } + + serializeStructuredField(fieldWriter, generator, field, value); + } + + public static <T extends FieldValue> void serializeWeightedSet(JsonGenerator generator, FieldBase field, WeightedSet<T> value) { + fieldNameIfNotNull(generator, field); + + wrapIOException(() -> { + generator.writeStartObject(); + + for (T key : value.keySet()) { + Integer weight = value.get(key); + // key.toString() is according to spec + generator.writeNumberField(key.toString(), weight); + } + + generator.writeEndObject(); + }); + } + + public static <T extends FieldValue> void serializeCollectionField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, CollectionFieldValue<T> value) { + fieldNameIfNotNull(generator, field); + + wrapIOException(() -> { + generator.writeStartArray(); + Iterator<T> i = value.iterator(); + + while (i.hasNext()) { + i.next().serialize(null, fieldWriter); + } + + generator.writeEndArray(); + }); + } + + + public static <K extends FieldValue, V extends FieldValue> void serializeMapField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, MapFieldValue<K, V> map) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> { + generator.writeStartArray(); + + for (Map.Entry<K, V> entry : map.entrySet()) { + generator.writeStartObject(); + generator.writeFieldName(JsonReader.MAP_KEY); + entry.getKey().serialize(null, fieldWriter); + generator.writeFieldName(JsonReader.MAP_VALUE); + entry.getValue().serialize(null, fieldWriter); + generator.writeEndObject(); + } + + generator.writeEndArray(); + }); + } + + public static <T extends FieldValue> void serializeArrayField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Array<T> value) { + wrapIOException(() -> { + fieldNameIfNotNull(generator, field); + generator.writeStartArray(); + + for (T elem : value) { + elem.serialize(null, fieldWriter); + } + + generator.writeEndArray(); + }); + } + + public static void serializeDoubleField(JsonGenerator generator, FieldBase field, DoubleFieldValue value) { + serializeDouble(generator, field, value.getDouble()); + } + + public static void serializeFloatField(JsonGenerator generator, FieldBase field, FloatFieldValue value) { + serializeFloat(generator, field, value.getFloat()); + } + + public static void serializeIntField(JsonGenerator generator, FieldBase field, IntegerFieldValue value) { + serializeInt(generator, field, value.getInteger()); + } + + public static void serializeLongField(JsonGenerator generator, FieldBase field, LongFieldValue value) { + serializeLong(generator, field, value.getLong()); + } + + public static void serializeByteField(JsonGenerator generator, FieldBase field, ByteFieldValue value) { + serializeByte(generator, field, value.getByte()); + } + + public static void serializePredicateField(JsonGenerator generator, FieldBase field, PredicateFieldValue value){ + serializeString(generator, field, value.toString()); + } + + public static void serializeRawField(JsonGenerator generator, FieldBase field, Raw raw) { + serializeByteBuffer(generator, field, raw.getByteBuffer()); + } + + public static void serializeString(JsonGenerator generator, FieldBase field, String value) { + if (value.length() == 0) { + return; + } + + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeString(value)); + } + + public static void serializeByte(JsonGenerator generator, FieldBase field, byte value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeShort(JsonGenerator generator, FieldBase field, short value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeInt(JsonGenerator generator, FieldBase field, int value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeLong(JsonGenerator generator, FieldBase field, long value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeFloat(JsonGenerator generator, FieldBase field, float value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeDouble(JsonGenerator generator, FieldBase field, double value) { + fieldNameIfNotNull(generator, field); + wrapIOException(() -> generator.writeNumber(value)); + } + + public static void serializeByteBuffer(JsonGenerator generator, FieldBase field, ByteBuffer raw) { + fieldNameIfNotNull(generator, field); + + final byte[] data = new byte[raw.remaining()]; + final int origPosition = raw.position(); + + // base64encoder has no encode methods with offset and + // limit, so no use trying to get at the backing array if + // available anyway + raw.get(data); + raw.position(origPosition); + + wrapIOException(() -> generator.writeString(base64Encoder.encodeToString(data))); + } + + public static void serializeByteArray(JsonGenerator generator, FieldBase field, byte[] value) { + serializeByteBuffer(generator, field, ByteBuffer.wrap(value)); + } + + public static void fieldNameIfNotNull(JsonGenerator generator, FieldBase field) { + if (field != null) { + wrapIOException(() -> generator.writeFieldName(field.getName())); + } + } +} diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index 79a1c040cbc..626c97a958a 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -1,34 +1,43 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.json; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import com.yahoo.document.datatypes.*; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorAddress; -import org.apache.commons.codec.binary.Base64; - import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; -import com.yahoo.document.PositionDataType; import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.document.serialization.DocumentWriter; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.vespa.objects.Serializer; - import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.Map; + +import static com.yahoo.document.json.JsonSerializationHelper.*; + /** * Serialize Document and other FieldValue instances as JSON. * @@ -38,7 +47,6 @@ public class JsonWriter implements DocumentWriter { private static final JsonFactory jsonFactory = new JsonFactory(); private final JsonGenerator generator; - private final Base64 base64Encoder = new Base64(); // I really hate exception unsafe constructors, but the alternative // requires generator to not be a final @@ -85,22 +93,23 @@ public class JsonWriter implements DocumentWriter { */ @Override public void write(FieldBase field, FieldValue value) { - throw new UnsupportedOperationException("Serializing " - + value.getClass().getName() + " is not supported."); + throw new UnsupportedOperationException("Serializing " + value.getClass().getName() + " is not supported."); } @Override public void write(FieldBase field, Document value) { try { - fieldNameIfNotNull(field); + fieldNameIfNotNull(generator, field); generator.writeStartObject(); + // this makes it impossible to refeed directly, not sure what's correct // perhaps just change to "put"? generator.writeStringField("id", value.getId().toString()); generator.writeObjectFieldStart(JsonReader.FIELDS); - for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i - .hasNext();) { - Entry<Field, FieldValue> entry = i.next(); + + Iterator<Map.Entry<Field, FieldValue>> i = value.iterator(); + while (i.hasNext()) { + Map.Entry<Field, FieldValue> entry = i.next(); entry.getValue().serialize(entry.getKey(), this); } generator.writeEndObject(); @@ -113,199 +122,77 @@ public class JsonWriter implements DocumentWriter { @Override public <T extends FieldValue> void write(FieldBase field, Array<T> value) { - try { - fieldNameIfNotNull(field); - generator.writeStartArray(); - for (Iterator<T> i = value.iterator(); i.hasNext();) { - i.next().serialize(null, this); - } - generator.writeEndArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - } - - private void fieldNameIfNotNull(FieldBase field) { - if (field != null) { - try { - generator.writeFieldName(field.getName()); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + serializeArrayField(this, generator, field, value); } @Override - public <K extends FieldValue, V extends FieldValue> void write( - FieldBase field, MapFieldValue<K, V> map) { - fieldNameIfNotNull(field); - try { - generator.writeStartArray(); - for (Map.Entry<K, V> entry : map.entrySet()) { - generator.writeStartObject(); - generator.writeFieldName(JsonReader.MAP_KEY); - entry.getKey().serialize(null, this); - generator.writeFieldName(JsonReader.MAP_VALUE); - entry.getValue().serialize(null, this); - generator.writeEndObject(); - } - generator.writeEndArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } + public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) { + serializeMapField(this, generator, field, map); } @Override public void write(FieldBase field, ByteFieldValue value) { - putByte(field, value.getByte()); + serializeByteField(generator, field, value); } @Override - public <T extends FieldValue> void write(FieldBase field, - CollectionFieldValue<T> value) { - fieldNameIfNotNull(field); - try { - generator.writeStartArray(); - for (Iterator<T> i = value.iterator(); i.hasNext();) { - i.next().serialize(null, this); - } - generator.writeEndArray(); - } catch (IOException e) { - throw new RuntimeException(e); - } + public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { + serializeCollectionField(this, generator, field, value); } @Override public void write(FieldBase field, DoubleFieldValue value) { - putDouble(field, value.getDouble()); + serializeDoubleField(generator, field, value); } @Override public void write(FieldBase field, FloatFieldValue value) { - putFloat(field, value.getFloat()); + serializeFloatField(generator, field, value); } @Override public void write(FieldBase field, IntegerFieldValue value) { - putInt(field, value.getInteger()); + serializeIntField(generator, field, value); } @Override public void write(FieldBase field, LongFieldValue value) { - putLong(field, value.getLong()); + serializeLongField(generator, field, value); } @Override public void write(FieldBase field, Raw value) { - put(field, value.getByteBuffer()); + serializeRawField(generator, field, value); } @Override public void write(FieldBase field, PredicateFieldValue value) { - put(field, value.toString()); + serializePredicateField(generator, field, value); } @Override public void write(FieldBase field, StringFieldValue value) { - put(field, value.getString()); + serializeStringField(generator, field, value); } @Override public void write(FieldBase field, TensorFieldValue value) { - try { - fieldNameIfNotNull(field); - generator.writeStartObject(); - if (value.getTensor().isPresent()) { - Tensor tensor = value.getTensor().get(); - writeTensorDimensions(tensor.dimensions()); - writeTensorCells(tensor.cells()); - } - generator.writeEndObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private void writeTensorDimensions(Set<String> dimensions) throws IOException { - generator.writeArrayFieldStart(JsonReader.TENSOR_DIMENSIONS); - for (String dimension : dimensions) { - generator.writeString(dimension); - } - generator.writeEndArray(); - } - - private void writeTensorCells(Map<TensorAddress, Double> cells) throws IOException { - generator.writeArrayFieldStart(JsonReader.TENSOR_CELLS); - for (Map.Entry<TensorAddress, Double> cell : cells.entrySet()) { - generator.writeStartObject(); - writeTensorAddress(cell.getKey()); - generator.writeNumberField(JsonReader.TENSOR_VALUE, cell.getValue()); - generator.writeEndObject(); - } - generator.writeEndArray(); - } - - private void writeTensorAddress(TensorAddress address) throws IOException { - generator.writeObjectFieldStart(JsonReader.TENSOR_ADDRESS); - for (TensorAddress.Element element : address.elements()) { - generator.writeStringField(element.dimension(), element.label()); - } - generator.writeEndObject(); + serializeTensorField(generator, field, value); } @Override public void write(FieldBase field, Struct value) { - if (value.getDataType() == PositionDataType.INSTANCE) { - put(field, PositionDataType.renderAsString(value)); - return; - } - fieldNameIfNotNull(field); - try { - generator.writeStartObject(); - for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i - .hasNext();) { - Entry<Field, FieldValue> entry = i.next(); - entry.getValue().serialize(entry.getKey(), this); - } - generator.writeEndObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } + serializeStructField(this, generator, field, value); } @Override public void write(FieldBase field, StructuredFieldValue value) { - fieldNameIfNotNull(field); - try { - generator.writeStartObject(); - for (Iterator<Entry<Field, FieldValue>> i = value.iterator(); i - .hasNext();) { - Entry<Field, FieldValue> entry = i.next(); - entry.getValue().serialize(entry.getKey(), this); - } - generator.writeEndObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } + serializeStructuredField(this, generator, field, value); } @Override - public <T extends FieldValue> void write(FieldBase field, - WeightedSet<T> value) { - fieldNameIfNotNull(field); - try { - generator.writeStartObject(); - // entrySet() is deprecated and there is no entry iterator - for (T key : value.keySet()) { - Integer weight = value.get(key); - // key.toString() is according to spec - generator.writeNumberField(key.toString(), weight); - } - generator.writeEndObject(); - } catch (IOException e) { - throw new RuntimeException(e); - } + public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> value) { + serializeWeightedSet(generator, field, value); } @Override @@ -315,110 +202,6 @@ public class JsonWriter implements DocumentWriter { } @Override - public Serializer putByte(FieldBase field, byte value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putShort(FieldBase field, short value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putInt(FieldBase field, int value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putLong(FieldBase field, long value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putFloat(FieldBase field, float value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer putDouble(FieldBase field, double value) { - fieldNameIfNotNull(field); - try { - generator.writeNumber(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer put(FieldBase field, byte[] value) { - return put(field, ByteBuffer.wrap(value)); - } - - @Override - public Serializer put(FieldBase field, ByteBuffer raw) { - final byte[] data = new byte[raw.remaining()]; - final int origPosition = raw.position(); - - fieldNameIfNotNull(field); - // base64encoder has no encode methods with offset and - // limit, so no use trying to get at the backing array if - // available anyway - raw.get(data); - raw.position(origPosition); - try { - generator.writeString(base64Encoder.encodeToString(data)); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override - public Serializer put(FieldBase field, String value) { - if (value.length() == 0) { - return this; - } - fieldNameIfNotNull(field); - try { - generator.writeString(value); - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - @Override public void write(Document document) { write(null, document); } @@ -470,4 +253,58 @@ public class JsonWriter implements DocumentWriter { } return out.toByteArray(); } + + @Override + public Serializer putByte(FieldBase field, byte value) { + serializeByte(generator, field, value); + return this; + } + + @Override + public Serializer putShort(FieldBase field, short value) { + serializeShort(generator, field, value); + return this; + } + + @Override + public Serializer putInt(FieldBase field, int value) { + serializeInt(generator, field, value); + return this; + } + + @Override + public Serializer putLong(FieldBase field, long value) { + serializeLong(generator, field, value); + return this; + } + + @Override + public Serializer putFloat(FieldBase field, float value) { + serializeFloat(generator, field, value); + return this; + } + + @Override + public Serializer putDouble(FieldBase field, double value) { + serializeDouble(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, byte[] value) { + serializeByteArray(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, ByteBuffer value) { + serializeByteBuffer(generator, field, value); + return this; + } + + @Override + public Serializer put(FieldBase field, String value) { + serializeString(generator, field, value); + return this; + } } diff --git a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java index 4a269a704d2..244aa0d5a0d 100644 --- a/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/FieldWriter.java @@ -3,7 +3,22 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; import com.yahoo.document.annotation.AnnotationReference; -import com.yahoo.document.datatypes.*; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.vespa.objects.Serializer; @@ -189,5 +204,4 @@ public interface FieldWriter extends Serializer { * field value */ void write(FieldBase field, AnnotationReference value); - } diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java index ffe6073cd6c..dc605fbe3d5 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlDocumentWriter.java @@ -1,21 +1,36 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.Iterator; -import java.util.Map; - import com.yahoo.document.Document; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.annotation.AnnotationReference; -import com.yahoo.document.datatypes.*; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.vespa.objects.Serializer; +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.Map; + // TODO: Just inline all use of XmlSerializationHelper when the toXml methods in FieldValue subclasses are to be removed // TODO: More cleanup, the put() methods generate a lot of superfluous objects (write should call put, not the other way around) // TODO: remove pingpong between XmlSerializationHelper and FieldValue, this will go away when the toXml methods go away |