diff options
8 files changed, 31 insertions, 780 deletions
diff --git a/document/abi-spec.json b/document/abi-spec.json index 62f0b67ea47..d6ee31695da 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -3622,8 +3622,7 @@ "public void <init>()", "public static com.yahoo.document.serialization.DocumentSerializer createHead(com.yahoo.io.GrowableByteBuffer)", "public static com.yahoo.document.serialization.DocumentSerializer create6(com.yahoo.io.GrowableByteBuffer)", - "public static com.yahoo.document.serialization.DocumentSerializer create6()", - "public static com.yahoo.document.serialization.DocumentSerializer create42(com.yahoo.io.GrowableByteBuffer)" + "public static com.yahoo.document.serialization.DocumentSerializer create6()" ], "fields": [] }, @@ -3908,59 +3907,6 @@ ], "fields": [] }, - "com.yahoo.document.serialization.VespaDocumentSerializer42": { - "superClass": "com.yahoo.vespa.objects.BufferSerializer", - "interfaces": [ - "com.yahoo.document.serialization.DocumentSerializer" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void write(com.yahoo.document.Document)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.Document)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.IntegerFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.LongFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Raw)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.PredicateFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StringFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.TensorFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ReferenceFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Struct)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StructuredFieldValue)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.WeightedSet)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.annotation.AnnotationReference)", - "public void write(com.yahoo.document.DocumentId)", - "public void write(com.yahoo.document.DocumentType)", - "public void write(com.yahoo.document.annotation.Annotation)", - "public void write(com.yahoo.document.annotation.SpanTree)", - "public void write(com.yahoo.document.annotation.SpanNode)", - "public void write(com.yahoo.document.annotation.Span)", - "public void write(com.yahoo.document.annotation.SpanList)", - "public void write(com.yahoo.document.annotation.AlternateSpanList)", - "public void write(com.yahoo.document.DocumentUpdate)", - "public void write(com.yahoo.document.update.FieldUpdate)", - "public void write(com.yahoo.document.update.AddValueUpdate, com.yahoo.document.DataType)", - "public void write(com.yahoo.document.update.MapValueUpdate, com.yahoo.document.DataType)", - "public void write(com.yahoo.document.update.ArithmeticValueUpdate)", - "public void write(com.yahoo.document.update.AssignValueUpdate, com.yahoo.document.DataType)", - "public void write(com.yahoo.document.update.RemoveValueUpdate, com.yahoo.document.DataType)", - "public void write(com.yahoo.document.update.ClearValueUpdate, com.yahoo.document.DataType)", - "public void write(com.yahoo.document.update.TensorModifyUpdate)", - "public void write(com.yahoo.document.update.TensorAddUpdate)", - "public void write(com.yahoo.document.update.TensorRemoveUpdate)", - "public static long getSerializedSize(com.yahoo.document.Document)" - ], - "fields": [] - }, "com.yahoo.document.serialization.VespaDocumentSerializer6": { "superClass": "com.yahoo.vespa.objects.BufferSerializer", "interfaces": [ diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java index 23266566e1c..2120353a61d 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java @@ -34,12 +34,4 @@ public class DocumentSerializerFactory { return new VespaDocumentSerializer6(new GrowableByteBuffer()); } - /** - * Creates a serializer for the document format that was created on Vespa 4.2. - */ - @SuppressWarnings("deprecation") - public static DocumentSerializer create42(GrowableByteBuffer buf) { - return new VespaDocumentSerializer42(buf); - } - } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java deleted file mode 100644 index 746b3c9d68d..00000000000 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java +++ /dev/null @@ -1,689 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.serialization; - -import com.yahoo.compress.Compressor; - -import com.yahoo.document.ArrayDataType; -import com.yahoo.document.CollectionDataType; -import com.yahoo.document.DataType; -import com.yahoo.document.Document; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentType; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.Field; -import com.yahoo.document.WeightedSetDataType; -import com.yahoo.document.annotation.AlternateSpanList; -import com.yahoo.document.annotation.Annotation; -import com.yahoo.document.annotation.AnnotationReference; -import com.yahoo.document.annotation.Span; -import com.yahoo.document.annotation.SpanList; -import com.yahoo.document.annotation.SpanNode; -import com.yahoo.document.annotation.SpanTree; -import com.yahoo.document.datatypes.Array; -import com.yahoo.document.datatypes.BoolFieldValue; -import com.yahoo.document.datatypes.ByteFieldValue; -import com.yahoo.document.datatypes.CollectionFieldValue; -import com.yahoo.document.datatypes.DoubleFieldValue; -import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.document.datatypes.FloatFieldValue; -import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.LongFieldValue; -import com.yahoo.document.datatypes.MapFieldValue; -import com.yahoo.document.datatypes.PredicateFieldValue; -import com.yahoo.document.datatypes.Raw; -import com.yahoo.document.datatypes.ReferenceFieldValue; -import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.datatypes.Struct; -import com.yahoo.document.datatypes.StructuredFieldValue; -import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.document.datatypes.WeightedSet; -import com.yahoo.document.predicate.BinaryFormat; -import com.yahoo.document.update.AddValueUpdate; -import com.yahoo.document.update.ArithmeticValueUpdate; -import com.yahoo.document.update.AssignValueUpdate; -import com.yahoo.document.update.ClearValueUpdate; -import com.yahoo.document.update.FieldUpdate; -import com.yahoo.document.update.MapValueUpdate; -import com.yahoo.document.update.RemoveValueUpdate; -import com.yahoo.document.update.TensorAddUpdate; -import com.yahoo.document.update.TensorModifyUpdate; -import com.yahoo.document.update.TensorRemoveUpdate; -import com.yahoo.document.update.ValueUpdate; -import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.tensor.serialization.TypedBinaryFormat; -import com.yahoo.vespa.objects.BufferSerializer; -import com.yahoo.vespa.objects.FieldBase; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import static com.yahoo.text.Utf8.calculateBytePositions; - -/** - * Class used for serializing documents on the Vespa 4.2 document format. - * - * @deprecated use {@link com.yahoo.document.serialization.VespaDocumentSerializerHead} instead for new code - * @author baldersheim - */ -@Deprecated // TODO: Remove on Vespa 8 -// When removing: Move content into VespaDocumentSerializerHead -public class VespaDocumentSerializer42 extends BufferSerializer implements DocumentSerializer { - - private int spanNodeCounter = -1; - private int[] bytePositions; - - VespaDocumentSerializer42(GrowableByteBuffer buf) { - super(buf); - } - - VespaDocumentSerializer42() { - super(); - } - - public void write(Document doc) { - write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc); - } - - public void write(FieldBase field, Document doc) { - buf.putShort(Document.SERIALIZED_VERSION); - - //save the starting position in the buffer - int lenPos = buf.position(); - // Temporary length, fill in after serialization is done. - buf.putInt(0); - - doc.getId().serialize(this); - - Struct head = doc.getHeader(); - Struct body = doc.getBody(); - boolean hasHead = (head.getFieldCount() != 0); - boolean hasBody = (body.getFieldCount() != 0); - - byte contents = 0x01; // Indicating we have document type which we always have - if (hasHead) { - contents |= 0x2; // Indicate we have header - } - if (hasBody) { - contents |= 0x4; // Indicate we have a body - } - buf.put(contents); - - doc.getDataType().serialize(this); - if (hasHead) { - head.serialize(null, this); - } - if (hasBody) { - body.serialize(null, this); - } - int finalPos = buf.position(); - buf.position(lenPos); - buf.putInt(finalPos - lenPos - 4); // Don't include the length itself or the version - buf.position(finalPos); - } - - /** - * Write out the value of field - * - * @param field - field description (name and data type) - * @param value - field value - */ - public void write(FieldBase field, FieldValue value) { - throw new IllegalArgumentException("Not Implemented"); - } - - /** - * Write out the value of array field - * - * @param field - field description (name and data type) - * @param array - field value - */ - public <T extends FieldValue> void write(FieldBase field, Array<T> array) { - buf.putInt1_2_4Bytes(array.size()); - - List<T> lst = array.getValues(); - for (FieldValue value : lst) { - value.serialize(this); - } - - } - - public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) { - buf.putInt1_2_4Bytes(map.size()); - for (Map.Entry<K, V> e : map.entrySet()) { - e.getKey().serialize(this); - e.getValue().serialize(this); - } - } - - @Override - public void write(FieldBase field, ByteFieldValue value) { - buf.put(value.getByte()); - } - - @Override - public void write(FieldBase field, BoolFieldValue value) { - byte v = value.getBoolean() ? (byte)1 : (byte)0; - buf.put(v); - } - - /** - * Write out the value of collection field - * - * @param field - field description (name and data type) - * @param value - field value - */ - public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { - throw new IllegalArgumentException("Not Implemented"); - } - - /** - * Write out the value of double field - * - * @param field - field description (name and data type) - * @param value - field value - */ - public void write(FieldBase field, DoubleFieldValue value) { - buf.putDouble(value.getDouble()); - } - - /** - * Write out the value of float field - * - * @param field - field description (name and data type) - * @param value - field value - */ - public void write(FieldBase field, FloatFieldValue value) { - buf.putFloat(value.getFloat()); - } - - /** - * Write out the value of integer field - * - * @param field - field description (name and data type) - * @param value - field value - */ - public void write(FieldBase field, IntegerFieldValue value) { - buf.putInt(value.getInteger()); - } - - /** - * Write out the value of long field - * - * @param field - field description (name and data type) - * @param value - field value - */ - public void write(FieldBase field, LongFieldValue value) { - buf.putLong(value.getLong()); - } - - /** - * Write out the value of raw field - * - * @param field - field description (name and data type) - * @param value - field value - */ - public void write(FieldBase field, Raw value) { - ByteBuffer rawBuf = value.getByteBuffer(); - int origPos = rawBuf.position(); - buf.putInt(rawBuf.remaining()); - buf.put(rawBuf); - rawBuf.position(origPos); - - } - - @Override - public void write(FieldBase field, PredicateFieldValue value) { - byte[] buf = BinaryFormat.encode(value.getPredicate()); - this.buf.putInt(buf.length); - this.buf.put(buf); - } - - /** - * Write out the value of string field - * - * @param field - field description (name and data type) - * @param value - field value - */ - public void write(FieldBase field, StringFieldValue value) { - byte[] stringBytes = createUTF8CharArray(value.getString()); - - byte coding = 0; - //Use bit 6 of "coding" to say whether span tree is available or not - if (!value.getSpanTrees().isEmpty()) { - coding |= 64; - } - buf.put(coding); - buf.putInt1_4Bytes(stringBytes.length + 1); - - buf.put(stringBytes); - buf.put(((byte) 0)); - - Map<String, SpanTree> trees = value.getSpanTreeMap(); - if ((trees != null) && !trees.isEmpty()) { - try { - //we don't support serialization of nested span trees, so this is safe: - bytePositions = calculateBytePositions(value.getString()); - //total length. record position and go back here if necessary: - int posBeforeSize = buf.position(); - buf.putInt(0); - buf.putInt1_2_4Bytes(trees.size()); - - for (SpanTree tree : trees.values()) { - try { - write(tree); - } catch (SerializationException e) { - throw e; - } catch (RuntimeException e) { - throw new SerializationException("Exception thrown while serializing span tree '" + - tree.getName() + "'; string='" + value.getString() + "'", e); - } - } - int endPos = buf.position(); - buf.position(posBeforeSize); - buf.putInt(endPos - posBeforeSize - 4); //length shall exclude itself - buf.position(endPos); - } finally { - bytePositions = null; - } - } - } - - @Override - public void write(FieldBase field, TensorFieldValue value) { - if (value.getTensor().isPresent()) { - byte[] encodedTensor = TypedBinaryFormat.encode(value.getTensor().get()); - buf.putInt1_4Bytes(encodedTensor.length); - buf.put(encodedTensor); - } else { - buf.putInt1_4Bytes(0); - } - } - - @Override - public void write(FieldBase field, ReferenceFieldValue value) { - if (value.getDocumentId().isPresent()) { - // We piggyback on DocumentId's existing serialization code, but need to know - // whether or not it's present or merely the empty string. - buf.put((byte)1); - write(value.getDocumentId().get()); - } else { - buf.put((byte)0); - } - } - - /** - * Write out the value of struct field - * - * @param field - field description (name and data type) - * @param s - field value - */ - public void write(FieldBase field, Struct s) { - // Serialize all parts first.. As we need to know length before starting - // Serialize all the fields. - - //keep the buffer we're serializing everything into: - GrowableByteBuffer bigBuffer = buf; - - //create a new buffer and serialize into that for a while: - GrowableByteBuffer buffer = new GrowableByteBuffer(4096, 2.0f); - buf = buffer; - - List<Integer> fieldIds = new LinkedList<>(); - List<java.lang.Integer> fieldLengths = new LinkedList<>(); - - for (Map.Entry<Field, FieldValue> value : s.getFields()) { - - int startPos = buffer.position(); - value.getValue().serialize(value.getKey(), this); - - fieldLengths.add(buffer.position() - startPos); - fieldIds.add(value.getKey().getId(s.getVersion())); - } - - // Switch buffers again: - buffer.flip(); - buf = bigBuffer; - - int uncompressedSize = buffer.remaining(); - Compressor.Compression compression = - s.getDataType().getCompressor().compress(buffer.getByteBuffer().array(), buffer.remaining()); - - // Actual serialization starts here. - int lenPos = buf.position(); - putInt(null, 0); // Move back to this after compression is done. - buf.put(compression.type().getCode()); - - if (compression.data() != null && compression.type().isCompressed()) { - buf.putInt2_4_8Bytes(uncompressedSize); - } - - buf.putInt1_4Bytes(s.getFieldCount()); - - for (int i = 0; i < s.getFieldCount(); ++i) { - putInt1_4Bytes(null, fieldIds.get(i)); - putInt2_4_8Bytes(null, fieldLengths.get(i)); - } - - int pos = buf.position(); - if (compression.data() != null && compression.type().isCompressed()) { - put(null, compression.data()); - } else { - put(null, buffer.getByteBuffer()); - } - int dataLength = buf.position() - pos; - - int posNow = buf.position(); - buf.position(lenPos); - putInt(null, dataLength); - buf.position(posNow); - } - - /** - * Write out the value of structured field - * - * @param field - field description (name and data type) - * @param value - field value - */ - public void write(FieldBase field, StructuredFieldValue value) { - throw new IllegalArgumentException("Not Implemented"); - } - - /** - * Write out the value of weighted set field - * - * @param field - field description (name and data type) - * @param ws - field value - */ - public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> ws) { - WeightedSetDataType type = ws.getDataType(); - putInt(null, type.getNestedType().getId()); - putInt(null, ws.size()); - - Iterator<T> it = ws.fieldValueIterator(); - while (it.hasNext()) { - FieldValue key = it.next(); - java.lang.Integer value = ws.get(key); - int sizePos = buf.position(); - putInt(null, 0); - int startPos = buf.position(); - key.serialize(this); - putInt(null, value); - int finalPos = buf.position(); - int size = finalPos - startPos; - buf.position(sizePos); - putInt(null, size); - buf.position(finalPos); - } - - } - - public void write(FieldBase field, AnnotationReference value) { - int annotationId = value.getReference().getScratchId(); - if (annotationId >= 0) { - buf.putInt1_2_4Bytes(annotationId); - } else { - throw new SerializationException("Could not serialize AnnotationReference value, reference not found (" + value + ")"); - } - } - - public void write(DocumentId id) { - put(null, id.getScheme().toUtf8().getBytes()); - putByte(null, (byte) 0); - } - - public void write(DocumentType type) { - byte[] docType = createUTF8CharArray(type.getName()); - put(null, docType); - putByte(null, ((byte) 0)); - putShort(null, (short) 0); // Used to hold the version. Is now always 0. - } - - public void write(Annotation annotation) { - buf.putInt(annotation.getType().getId()); //name hash - - byte features = 0; - if (annotation.isSpanNodeValid()) { - features |= ((byte) 1); - } - if (annotation.hasFieldValue()) { - features |= ((byte) 2); - } - buf.put(features); - - int posBeforeSize = buf.position(); - buf.putInt1_2_4BytesAs4(0); - - //write ID of span node: - if (annotation.isSpanNodeValid()) { - int spanNodeId = annotation.getSpanNode().getScratchId(); - if (spanNodeId >= 0) { - buf.putInt1_2_4Bytes(spanNodeId); - } else { - throw new SerializationException("Could not serialize annotation, associated SpanNode not found (" + annotation + ")"); - } - } - - //write annotation value: - if (annotation.hasFieldValue()) { - buf.putInt(annotation.getType().getDataType().getId()); - annotation.getFieldValue().serialize(this); - } - - int end = buf.position(); - buf.position(posBeforeSize); - buf.putInt1_2_4BytesAs4(end - posBeforeSize - 4); - buf.position(end); - } - - public void write(SpanTree tree) { - //we don't support serialization of nested span trees: - if (spanNodeCounter >= 0) { - throw new SerializationException("Serialization of nested SpanTrees is not supported."); - } - - //we're going to write a new SpanTree, create a new Map for nodes: - spanNodeCounter = 0; - - //make sure tree is consistent before continuing: - tree.cleanup(); - - try { - new StringFieldValue(tree.getName()).serialize(this); - - write(tree.getRoot()); - { - //add all annotations to temporary list and sort it, to get predictable serialization - List<Annotation> tmpAnnotationList = new ArrayList<Annotation>(tree.numAnnotations()); - for (Annotation annotation : tree) { - tmpAnnotationList.add(annotation); - } - Collections.sort(tmpAnnotationList); - - int annotationCounter = 0; - //add all annotations to map here, in case of back-references: - for (Annotation annotation : tmpAnnotationList) { - annotation.setScratchId(annotationCounter++); - } - - buf.putInt1_2_4Bytes(tmpAnnotationList.size()); - for (Annotation annotation : tmpAnnotationList) { - write(annotation); - } - } - } finally { - //we're done, let's set these to null to save memory and prevent madness: - spanNodeCounter = -1; - } - } - - public void write(SpanNode spanNode) { - if (spanNodeCounter >= 0) { - spanNode.setScratchId(spanNodeCounter++); - } - if (spanNode instanceof Span) { - write((Span) spanNode); - } else if (spanNode instanceof AlternateSpanList) { - write((AlternateSpanList) spanNode); - } else if (spanNode instanceof SpanList) { - write((SpanList) spanNode); - } else { - throw new IllegalStateException("BUG!! Unable to serialize " + spanNode); - } - } - - public void write(Span span) { - buf.put(Span.ID); - - if (bytePositions != null) { - int byteFrom = bytePositions[span.getFrom()]; - int byteLength = bytePositions[span.getFrom() + span.getLength()] - byteFrom; - - buf.putInt1_2_4Bytes(byteFrom); - buf.putInt1_2_4Bytes(byteLength); - } else { - throw new SerializationException("Cannot serialize Span " + span + ", no access to parent StringFieldValue."); - } - } - - public void write(SpanList spanList) { - buf.put(SpanList.ID); - buf.putInt1_2_4Bytes(spanList.numChildren()); - Iterator<SpanNode> children = spanList.childIterator(); - while (children.hasNext()) { - write(children.next()); - } - } - - public void write(AlternateSpanList altSpanList) { - buf.put(AlternateSpanList.ID); - buf.putInt1_2_4Bytes(altSpanList.getNumSubTrees()); - for (int i = 0; i < altSpanList.getNumSubTrees(); i++) { - buf.putDouble(altSpanList.getProbability(i)); - buf.putInt1_2_4Bytes(altSpanList.numChildren(i)); - Iterator<SpanNode> children = altSpanList.childIterator(i); - while (children.hasNext()) { - write(children.next()); - } - } - } - - @Override - public void write(DocumentUpdate update) { - putShort(null, Document.SERIALIZED_VERSION); - update.getId().serialize(this); - - byte contents = 0x1; // Legacy to say we have document type - putByte(null, contents); - update.getDocumentType().serialize(this); - - putInt(null, update.fieldUpdates().size()); - - for (FieldUpdate up : update.fieldUpdates()) { - up.serialize(this); - } - } - - @Override - public void write(FieldUpdate update) { - putInt(null, update.getField().getId(Document.SERIALIZED_VERSION)); - putInt(null, update.getValueUpdates().size()); - for (ValueUpdate vupd : update.getValueUpdates()) { - putInt(null, vupd.getValueUpdateClassID().id); - vupd.serialize(this, update.getField().getDataType()); - } - } - - @Override - public void write(AddValueUpdate update, DataType superType) { - writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); - putInt(null, update.getWeight()); - } - - @Override - public void write(MapValueUpdate update, DataType superType) { - if (superType instanceof ArrayDataType) { - CollectionDataType type = (CollectionDataType) superType; - IntegerFieldValue index = (IntegerFieldValue) update.getValue(); - index.serialize(this); - putInt(null, update.getUpdate().getValueUpdateClassID().id); - update.getUpdate().serialize(this, type.getNestedType()); - } else if (superType instanceof WeightedSetDataType) { - writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); - putInt(null, update.getUpdate().getValueUpdateClassID().id); - update.getUpdate().serialize(this, DataType.INT); - } else { - throw new SerializationException("MapValueUpdate only works for arrays and weighted sets"); - } - } - - @Override - public void write(ArithmeticValueUpdate update) { - putInt(null, update.getOperator().id); - putDouble(null, update.getOperand().doubleValue()); - } - - @Override - public void write(AssignValueUpdate update, DataType superType) { - if (update.getValue() == null) { - putByte(null, (byte) 0); - } else { - putByte(null, (byte) 1); - writeValue(this, superType, update.getValue()); - } - } - - @Override - public void write(RemoveValueUpdate update, DataType superType) { - writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); - } - - @Override - public void write(ClearValueUpdate clearValueUpdate, DataType superType) { - //TODO: This has never ever been implemented. Has this ever worked? - } - - @Override - public void write(TensorModifyUpdate update) { - throw new IllegalArgumentException("Write of TensorModifyUpdate not implemented for Vespa 4.2"); - } - - @Override - public void write(TensorAddUpdate update) { - throw new IllegalArgumentException("Write of TensorAddUpdate not implemented for Vespa 4.2"); - } - - @Override - public void write(TensorRemoveUpdate update) { - throw new IllegalArgumentException("Write of TensorRemoveUpdate not implemented for Vespa 4.2"); - } - - /** - * Returns the serialized size of the given {@link Document}. Please note that this method performs actual - * serialization of the document, but simply return the size of the final {@link GrowableByteBuffer}. If you need - * the buffer itself, do NOT use this method. - * - * @param doc The Document whose size to calculate. - * @return The size in bytes. - */ - public static long getSerializedSize(Document doc) { - DocumentSerializer serializer = new VespaDocumentSerializerHead(new GrowableByteBuffer()); - serializer.write(doc); - return serializer.getBuf().position(); - } - - private static void writeValue(VespaDocumentSerializer42 serializer, DataType dataType, Object value) { - FieldValue fieldValue; - if (value instanceof FieldValue) { - fieldValue = (FieldValue)value; - } else { - fieldValue = dataType.createFieldValue(value); - } - fieldValue.serialize(serializer); - } - -} diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index de7d72c018d..c381a093b4e 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -297,14 +297,13 @@ public class DocumentUpdateTestCase { } } - @Ignore @Test public void testSerialize() { docUp.addFieldUpdate(assignSingle); docUp.addFieldUpdate(addMultiList); GrowableByteBuffer buf = new GrowableByteBuffer(); - docUp.serialize(DocumentSerializerFactory.create42(buf)); + docUp.serialize(DocumentSerializerFactory.create6(buf)); buf.flip(); try { @@ -314,9 +313,7 @@ public class DocumentUpdateTestCase { } catch (Exception e) { } - assertEquals(2 // version - + (17 + 1) //docid id:ns:foobar:bar\0 - + 1 //contents + assertEquals((17 + 1) //docid id:ns:foobar:bar\0 + (6 + 1 + 2) //doctype foobar\0\0\0 + 4 //num field updates @@ -334,6 +331,7 @@ public class DocumentUpdateTestCase { + (4 //valueUpdateClassID + (4 + 4 + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1))))) //value + + 4 //num field path updates , buf.remaining()); DocumentUpdate docUpDeser = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buf)); @@ -406,7 +404,7 @@ public class DocumentUpdateTestCase { upd.addFieldUpdate(serAdd); GrowableByteBuffer buf = new GrowableByteBuffer(100, 2.0f); - upd.serialize(DocumentSerializerFactory.create42(buf)); + upd.serialize(DocumentSerializerFactory.create6(buf)); buf.flip(); writeBufferToFile(buf, "src/tests/data/serializeupdatejava.dat"); diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java index 653f121b2d4..91d360b5d2a 100644 --- a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java @@ -182,7 +182,7 @@ public class SerializeAnnotationsTestCase { assertEquals(serialized.limit(), serializedFromFile.limit()); StringFieldValue valueFromFile = new StringFieldValue(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, new GrowableByteBuffer(serializedFromFile)); + DocumentDeserializer deserializer = DocumentDeserializerFactory.createHead(docMan, new GrowableByteBuffer(serializedFromFile)); deserializer.read(null, valueFromFile); assertEquals(value, valueFromFile); } @@ -191,7 +191,7 @@ public class SerializeAnnotationsTestCase { fileName = PATH + fileName; //serialize our tree to buffer - VespaDocumentSerializer42 serializer = new VespaDocumentSerializer42(); + DocumentSerializer serializer = DocumentSerializerFactory.create6(); serializer.write(null, value); ByteBuffer serializedBuf = serializer.getBuf().getByteBuffer(); serializedBuf.flip(); diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java index 6effbc9a452..598369bae39 100644 --- a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java @@ -41,7 +41,7 @@ public class VespaDocumentSerializerTestCase { GrowableByteBuffer buf = new GrowableByteBuffer(); doc.serialize(buf); - assertEquals(buf.position(), VespaDocumentSerializer42.getSerializedSize(doc)); + assertEquals(buf.position(), VespaDocumentSerializerHead.getSerializedSize(doc)); } @Test diff --git a/document/src/tests/data/serializeupdatejava.dat b/document/src/tests/data/serializeupdatejava.dat Binary files differindex cef1ad96812..2b6755c88d6 100644 --- a/document/src/tests/data/serializeupdatejava.dat +++ b/document/src/tests/data/serializeupdatejava.dat diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java index 27684e20aff..3933b188dc6 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java @@ -5,16 +5,21 @@ import com.yahoo.component.Version; import com.yahoo.component.VersionSpecification; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.serialization.*; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentDeserializerFactory; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.log.LogLevel; import com.yahoo.messagebus.Routable; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * This class encapsulates the logic required to map routable type and version to a corresponding {@link @@ -49,21 +54,23 @@ final class RoutableRepository { log.log(LogLevel.ERROR, "Received empty byte array for deserialization."); return null; } + if (version.getMajor() < 5) { + log.log(LogLevel.ERROR,"Can not decode anything from (version " + version + "). Only major version 5 and up supported."); + return null; + } DocumentDeserializer in = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data)); int type = in.getInt(null); RoutableFactory factory = getFactory(version, type); if (factory == null) { - log.log(LogLevel.ERROR, "No routable factory found for routable type " + type + - " (version " + version + ")."); + log.log(LogLevel.ERROR,"No routable factory found for routable type " + type + " (version " + version + ")."); return null; } Routable ret = factory.decode(in, loadTypes); if (ret == null) { - log.log(LogLevel.ERROR, "Routable factory " + factory.getClass().getName() + " failed to deserialize " + - "routable of type " + type + " (version " + version + ")."); - log.log(LogLevel.ERROR, Arrays.toString(data)); + log.log(LogLevel.ERROR,"Routable factory " + factory.getClass().getName() + " failed to deserialize " + + "routable of type " + type + " (version " + version + ").\nData = " + Arrays.toString(data)); return null; } return ret; @@ -83,17 +90,14 @@ final class RoutableRepository { int type = obj.getType(); RoutableFactory factory = getFactory(version, type); if (factory == null) { - log.log(LogLevel.ERROR, "No routable factory found for routable type " + type + - " (version " + version + ")."); + log.log(LogLevel.ERROR,"No routable factory found for routable type " + type + " (version " + version + ")."); return new byte[0]; } - DocumentSerializer out; - - if (version.getMajor() >= 5) { - out = DocumentSerializerFactory.createHead(new GrowableByteBuffer(8192)); - } else { - out = DocumentSerializerFactory.create42(new GrowableByteBuffer(8192)); + if (version.getMajor() < 5) { + log.log(LogLevel.ERROR,"Can not encode routable type " + type + " (version " + version + "). Only major version 5 and up supported."); + return new byte[0]; } + DocumentSerializer out= DocumentSerializerFactory.createHead(new GrowableByteBuffer(8192)); out.putInt(null, type); if (!factory.encode(obj, out)) { @@ -133,7 +137,7 @@ final class RoutableRepository { * @param type The routable type that the factory must support. * @return The routable factory matching the criteria, or null. */ - RoutableFactory getFactory(Version version, int type) { + private RoutableFactory getFactory(Version version, int type) { CacheKey cacheKey = new CacheKey(version, type); RoutableFactory factory = cache.get(cacheKey); if (factory != null) { @@ -204,7 +208,7 @@ final class RoutableRepository { final Version version; final int type; - public CacheKey(Version version, int type) { + CacheKey(Version version, int type) { this.version = version; this.type = type; } |