diff options
| author | Geir Storli <geirstorli@yahoo.no> | 2019-01-21 16:06:53 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2019-01-21 16:06:53 +0100 |
| commit | 8372a883c5a5fa100f88fc9b80824359b5bb70cd (patch) | |
| tree | 660ce3ef61f060b33bd97eac1185715606b8e2bd /document | |
| parent | 3722c1cdd91fce30d1c2538b2a8749d9321e194b (diff) | |
| parent | eb0b1134a66507e3bd8f09793c22cd824d01dff5 (diff) | |
Merge pull request #8198 from vespa-engine/7
7 MERGEOK
Diffstat (limited to 'document')
48 files changed, 1860 insertions, 740 deletions
diff --git a/document/abi-spec.json b/document/abi-spec.json index 79e0cdc34d0..a17bc298289 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -295,8 +295,6 @@ "public java.lang.Long getLastModified()", "public void setLastModified(java.lang.Long)", "public void onSerialize(com.yahoo.vespa.objects.Serializer)", - "public void serializeHeader(com.yahoo.vespa.objects.Serializer)", - "public void serializeBody(com.yahoo.vespa.objects.Serializer)", "public com.yahoo.document.DocumentType getDataType()", "public int getFieldCount()", "public void serialize(com.yahoo.document.serialization.DocumentWriter)", @@ -508,7 +506,6 @@ "public com.yahoo.document.DocumentType getDocumentType(java.lang.String)", "public final com.yahoo.document.Document createDocument(com.yahoo.io.GrowableByteBuffer)", "public com.yahoo.document.Document createDocument(com.yahoo.document.serialization.DocumentDeserializer)", - "public com.yahoo.document.Document createDocument(com.yahoo.io.GrowableByteBuffer, com.yahoo.io.GrowableByteBuffer)", "public java.util.Collection getDataTypes()", "public java.util.Map getDocumentTypes()", "public java.util.Iterator documentTypeIterator()", @@ -553,14 +550,10 @@ "public void setId(com.yahoo.document.DocumentId)", "public com.yahoo.document.DocumentUpdate applyTo(com.yahoo.document.Document)", "public com.yahoo.document.DocumentUpdate prune(com.yahoo.document.Document)", - "public java.util.List getFieldUpdates()", "public java.util.Collection fieldUpdates()", - "public java.util.List getFieldPathUpdates()", "public java.util.Collection fieldPathUpdates()", "public com.yahoo.document.DocumentType getDocumentType()", "public void setDocumentType(com.yahoo.document.DocumentType)", - "public com.yahoo.document.update.FieldUpdate getFieldUpdate(int)", - "public com.yahoo.document.update.FieldUpdate setFieldUpdate(int, com.yahoo.document.update.FieldUpdate)", "public com.yahoo.document.update.FieldUpdate getFieldUpdate(com.yahoo.document.Field)", "public void clearFieldUpdates()", "public com.yahoo.document.update.FieldUpdate getFieldUpdate(java.lang.String)", @@ -571,7 +564,6 @@ "public com.yahoo.document.DocumentUpdate addFieldUpdate(com.yahoo.document.update.FieldUpdate)", "public com.yahoo.document.DocumentUpdate addFieldPathUpdate(com.yahoo.document.fieldpathupdate.FieldPathUpdate)", "public void addAll(com.yahoo.document.DocumentUpdate)", - "public com.yahoo.document.update.FieldUpdate removeFieldUpdate(int)", "public com.yahoo.document.update.FieldUpdate removeFieldUpdate(com.yahoo.document.Field)", "public com.yahoo.document.update.FieldUpdate removeFieldUpdate(java.lang.String)", "public com.yahoo.document.DocumentType getType()", @@ -3227,7 +3219,7 @@ "public void setNewValues(com.yahoo.document.datatypes.Array)", "public com.yahoo.document.datatypes.Array getNewValues()", "public com.yahoo.document.datatypes.FieldPathIteratorHandler getIteratorHandler(com.yahoo.document.Document)", - "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializerHead)", + "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializer6)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" @@ -3253,7 +3245,7 @@ "public void setRemoveIfZero(boolean)", "public void setCreateMissingPath(boolean)", "public boolean isArithmetic()", - "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializerHead)", + "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializer6)", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()", @@ -3307,7 +3299,7 @@ "public com.yahoo.document.select.DocumentSelector getWhereClause()", "public java.lang.String getOriginalWhereClause()", "public void applyTo(com.yahoo.document.Document)", - "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializerHead)", + "public void serialize(com.yahoo.document.serialization.VespaDocumentSerializer6)", "public static com.yahoo.document.fieldpathupdate.FieldPathUpdate create(com.yahoo.document.fieldpathupdate.FieldPathUpdate$Type, com.yahoo.document.DocumentType, com.yahoo.document.serialization.DocumentUpdateReader)", "public boolean equals(java.lang.Object)", "public int hashCode()", @@ -3608,211 +3600,6 @@ ], "fields": [] }, - "com.yahoo.document.select.BucketSelector": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.document.BucketIdFactory)", - "public com.yahoo.document.select.BucketSet getBucketList(java.lang.String)" - ], - "fields": [] - }, - "com.yahoo.document.select.BucketSet": { - "superClass": "java.util.HashSet", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.document.BucketId)", - "public void <init>(com.yahoo.document.select.BucketSet)", - "public com.yahoo.document.select.BucketSet intersection(com.yahoo.document.select.BucketSet)", - "public com.yahoo.document.select.BucketSet union(com.yahoo.document.select.BucketSet)" - ], - "fields": [] - }, - "com.yahoo.document.select.Context": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.document.DocumentOperation)", - "public com.yahoo.document.DocumentOperation getDocumentOperation()", - "public void setDocumentOperation(com.yahoo.document.DocumentOperation)", - "public java.util.Map getVariables()", - "public void setVariables(java.util.Map)" - ], - "fields": [] - }, - "com.yahoo.document.select.DocumentSelector": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(java.lang.String)", - "public com.yahoo.document.select.Result accepts(com.yahoo.document.DocumentOperation)", - "public com.yahoo.document.select.Result accepts(com.yahoo.document.select.Context)", - "public com.yahoo.document.select.ResultList getMatchingResultList(com.yahoo.document.DocumentOperation)", - "public com.yahoo.document.select.ResultList getMatchingResultList(com.yahoo.document.select.Context)", - "public java.lang.String toString()", - "public com.yahoo.document.select.OrderingSpecification getOrdering(int)", - "public void visit(com.yahoo.document.select.Visitor)" - ], - "fields": [] - }, - "com.yahoo.document.select.NowCheckVisitor": { - "superClass": "java.lang.Object", - "interfaces": [ - "com.yahoo.document.select.Visitor" - ], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public boolean requiresConversion()", - "public void visit(com.yahoo.document.select.rule.ArithmeticNode)", - "public void visit(com.yahoo.document.select.rule.AttributeNode)", - "public void visit(com.yahoo.document.select.rule.ComparisonNode)", - "public void visit(com.yahoo.document.select.rule.DocumentNode)", - "public void visit(com.yahoo.document.select.rule.EmbracedNode)", - "public void visit(com.yahoo.document.select.rule.IdNode)", - "public void visit(com.yahoo.document.select.rule.LiteralNode)", - "public void visit(com.yahoo.document.select.rule.LogicNode)", - "public void visit(com.yahoo.document.select.rule.NegationNode)", - "public void visit(com.yahoo.document.select.rule.NowNode)", - "public void visit(com.yahoo.document.select.rule.SearchColumnNode)", - "public void visit(com.yahoo.document.select.rule.VariableNode)" - ], - "fields": [] - }, - "com.yahoo.document.select.OrderingSpecification": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(int)", - "public void <init>(int, long, short, short)", - "public int getOrder()", - "public long getOrderingStart()", - "public short getWidthBits()", - "public short getDivisionBits()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", - "public java.lang.String toString()" - ], - "fields": [ - "public static int ASCENDING", - "public static int DESCENDING", - "public final int order", - "public final long orderingStart", - "public final short widthBits", - "public final short divisionBits" - ] - }, - "com.yahoo.document.select.Result": { - "superClass": "java.lang.Enum", - "interfaces": [], - "attributes": [ - "public", - "final", - "enum" - ], - "methods": [ - "public static com.yahoo.document.select.Result[] values()", - "public static com.yahoo.document.select.Result valueOf(java.lang.String)", - "public java.lang.String toString()", - "public static com.yahoo.document.select.Result invert(com.yahoo.document.select.Result)", - "public static com.yahoo.document.select.Result toResult(java.lang.Object)" - ], - "fields": [ - "public static final enum com.yahoo.document.select.Result TRUE", - "public static final enum com.yahoo.document.select.Result FALSE", - "public static final enum com.yahoo.document.select.Result INVALID" - ] - }, - "com.yahoo.document.select.ResultList$ResultPair": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public com.yahoo.document.datatypes.FieldPathIteratorHandler$VariableMap getVariables()", - "public com.yahoo.document.select.Result getResult()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.document.select.ResultList$VariableValue": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>(com.yahoo.document.datatypes.FieldPathIteratorHandler$VariableMap, java.lang.Object)", - "public com.yahoo.document.datatypes.FieldPathIteratorHandler$VariableMap getVariables()", - "public java.lang.Object getValue()", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.document.select.ResultList": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public void <init>()", - "public void <init>(com.yahoo.document.select.Result)", - "public void add(com.yahoo.document.datatypes.FieldPathIteratorHandler$VariableMap, com.yahoo.document.select.Result)", - "public java.util.List getResults()", - "public static com.yahoo.document.select.ResultList fromBoolean(boolean)", - "public com.yahoo.document.select.Result toResult()", - "public com.yahoo.document.select.ResultList combineAND(com.yahoo.document.select.ResultList)", - "public com.yahoo.document.select.ResultList combineOR(com.yahoo.document.select.ResultList)", - "public static com.yahoo.document.select.ResultList toResultList(java.lang.Object)", - "public java.lang.String toString()" - ], - "fields": [] - }, - "com.yahoo.document.select.Visitor": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "interface", - "abstract" - ], - "methods": [ - "public abstract void visit(com.yahoo.document.select.rule.ArithmeticNode)", - "public abstract void visit(com.yahoo.document.select.rule.AttributeNode)", - "public abstract void visit(com.yahoo.document.select.rule.ComparisonNode)", - "public abstract void visit(com.yahoo.document.select.rule.DocumentNode)", - "public abstract void visit(com.yahoo.document.select.rule.EmbracedNode)", - "public abstract void visit(com.yahoo.document.select.rule.IdNode)", - "public abstract void visit(com.yahoo.document.select.rule.LiteralNode)", - "public abstract void visit(com.yahoo.document.select.rule.LogicNode)", - "public abstract void visit(com.yahoo.document.select.rule.NegationNode)", - "public abstract void visit(com.yahoo.document.select.rule.NowNode)", - "public abstract void visit(com.yahoo.document.select.rule.SearchColumnNode)", - "public abstract void visit(com.yahoo.document.select.rule.VariableNode)" - ], - "fields": [] - }, "com.yahoo.document.select.convert.NowQueryExpression": { "superClass": "java.lang.Object", "interfaces": [], @@ -4598,8 +4385,8 @@ "methods": [ "public void <init>()", "public static com.yahoo.document.serialization.DocumentDeserializer createHead(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)", - "public static com.yahoo.document.serialization.DocumentDeserializer create42(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)", - "public static com.yahoo.document.serialization.DocumentDeserializer create42(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer, com.yahoo.io.GrowableByteBuffer)" + "public static com.yahoo.document.serialization.DocumentDeserializer create6(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)", + "public static com.yahoo.document.serialization.DocumentDeserializer create42(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)" ], "fields": [] }, @@ -4646,8 +4433,9 @@ "methods": [ "public void <init>()", "public static com.yahoo.document.serialization.DocumentSerializer createHead(com.yahoo.io.GrowableByteBuffer)", + "public static com.yahoo.document.serialization.DocumentSerializer create6(com.yahoo.io.GrowableByteBuffer)", + "public static com.yahoo.document.serialization.DocumentSerializer create6()", "public static com.yahoo.document.serialization.DocumentSerializer create42(com.yahoo.io.GrowableByteBuffer)", - "public static com.yahoo.document.serialization.DocumentSerializer create42(com.yahoo.io.GrowableByteBuffer, boolean)", "public static com.yahoo.document.serialization.DocumentSerializer create42()" ], "fields": [] @@ -4866,7 +4654,7 @@ "fields": [] }, "com.yahoo.document.serialization.VespaDocumentDeserializer42": { - "superClass": "com.yahoo.document.serialization.VespaDocumentSerializer42", + "superClass": "com.yahoo.vespa.objects.BufferSerializer", "interfaces": [ "com.yahoo.document.serialization.DocumentDeserializer" ], @@ -4913,16 +4701,62 @@ ], "fields": [] }, + "com.yahoo.document.serialization.VespaDocumentDeserializer6": { + "superClass": "com.yahoo.vespa.objects.BufferSerializer", + "interfaces": [ + "com.yahoo.document.serialization.DocumentDeserializer" + ], + "attributes": [ + "public" + ], + "methods": [ + "public final com.yahoo.document.DocumentTypeManager getDocumentTypeManager()", + "public void read(com.yahoo.document.Document)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.Document)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.IntegerFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.LongFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Raw)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.PredicateFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StringFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.TensorFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ReferenceFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Struct)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StructuredFieldValue)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.WeightedSet)", + "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.annotation.AnnotationReference)", + "public void read(com.yahoo.document.DocumentUpdate)", + "public void read(com.yahoo.document.fieldpathupdate.FieldPathUpdate)", + "public void read(com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate)", + "public void read(com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate)", + "public void read(com.yahoo.document.fieldpathupdate.AddFieldPathUpdate)", + "public com.yahoo.document.update.ValueUpdate getValueUpdate(com.yahoo.document.DataType, com.yahoo.document.DataType)", + "public void read(com.yahoo.document.update.FieldUpdate)", + "public com.yahoo.document.DocumentId readDocumentId()", + "public com.yahoo.document.DocumentType readDocumentType()", + "public void read(com.yahoo.document.annotation.SpanTree)", + "public void read(com.yahoo.document.annotation.Annotation)", + "public void read(com.yahoo.document.annotation.Span)", + "public void read(com.yahoo.document.annotation.SpanList)", + "public void read(com.yahoo.document.annotation.AlternateSpanList)" + ], + "fields": [] + }, "com.yahoo.document.serialization.VespaDocumentDeserializerHead": { - "superClass": "com.yahoo.document.serialization.VespaDocumentDeserializer42", + "superClass": "com.yahoo.document.serialization.VespaDocumentDeserializer6", "interfaces": [], "attributes": [ "public" ], "methods": [ - "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)", - "public void read(com.yahoo.document.DocumentUpdate)", - "public void read(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)" + "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.io.GrowableByteBuffer)" ], "fields": [] }, @@ -4935,7 +4769,6 @@ "public" ], "methods": [ - "public void setHeaderOnly(boolean)", "public void write(com.yahoo.document.Document)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.Document)", "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FieldValue)", @@ -4977,19 +4810,67 @@ ], "fields": [] }, - "com.yahoo.document.serialization.VespaDocumentSerializerHead": { - "superClass": "com.yahoo.document.serialization.VespaDocumentSerializer42", - "interfaces": [], + "com.yahoo.document.serialization.VespaDocumentSerializer6": { + "superClass": "com.yahoo.vespa.objects.BufferSerializer", + "interfaces": [ + "com.yahoo.document.serialization.DocumentSerializer" + ], "attributes": [ "public" ], "methods": [ - "public void <init>(com.yahoo.io.GrowableByteBuffer)", + "public void write(com.yahoo.document.Document)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.Document)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Array)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.MapFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.BoolFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.CollectionFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.DoubleFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.FloatFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.IntegerFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.LongFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Raw)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.PredicateFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StringFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.TensorFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ReferenceFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.Struct)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.StructuredFieldValue)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.WeightedSet)", + "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.annotation.AnnotationReference)", + "public void write(com.yahoo.document.DocumentId)", + "public void write(com.yahoo.document.DocumentType)", + "public void write(com.yahoo.document.annotation.Annotation)", + "public void write(com.yahoo.document.annotation.SpanTree)", + "public void write(com.yahoo.document.annotation.SpanNode)", + "public void write(com.yahoo.document.annotation.Span)", + "public void write(com.yahoo.document.annotation.SpanList)", + "public void write(com.yahoo.document.annotation.AlternateSpanList)", "public void write(com.yahoo.document.DocumentUpdate)", "public void write(com.yahoo.document.fieldpathupdate.FieldPathUpdate)", "public void write(com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate)", "public void write(com.yahoo.document.fieldpathupdate.AddFieldPathUpdate)", - "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.datatypes.ByteFieldValue)" + "public void write(com.yahoo.document.update.FieldUpdate)", + "public void write(com.yahoo.document.update.AddValueUpdate, com.yahoo.document.DataType)", + "public void write(com.yahoo.document.update.MapValueUpdate, com.yahoo.document.DataType)", + "public void write(com.yahoo.document.update.ArithmeticValueUpdate)", + "public void write(com.yahoo.document.update.AssignValueUpdate, com.yahoo.document.DataType)", + "public void write(com.yahoo.document.update.RemoveValueUpdate, com.yahoo.document.DataType)", + "public void write(com.yahoo.document.update.ClearValueUpdate, com.yahoo.document.DataType)", + "public static long getSerializedSize(com.yahoo.document.Document)" + ], + "fields": [] + }, + "com.yahoo.document.serialization.VespaDocumentSerializerHead": { + "superClass": "com.yahoo.document.serialization.VespaDocumentSerializer6", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.io.GrowableByteBuffer)" ], "fields": [] }, @@ -5359,4 +5240,4 @@ "protected com.yahoo.document.update.ValueUpdate$ValueUpdateClassID valueUpdateClassID" ] } -}
\ No newline at end of file +} diff --git a/document/pom.xml b/document/pom.xml index b8344391631..19f1a9470fc 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -8,11 +8,11 @@ <parent> <groupId>com.yahoo.vespa</groupId> <artifactId>parent</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>document</artifactId> - <version>6-SNAPSHOT</version> + <version>7-SNAPSHOT</version> <packaging>jar</packaging> <name>${project.artifactId}</name> <dependencies> diff --git a/document/src/main/java/com/yahoo/document/Document.java b/document/src/main/java/com/yahoo/document/Document.java index 222ebe29c6d..cf0951fb035 100644 --- a/document/src/main/java/com/yahoo/document/Document.java +++ b/document/src/main/java/com/yahoo/document/Document.java @@ -137,7 +137,7 @@ public class Document extends StructuredFieldValue { } public int getSerializedSize() throws SerializationException { - DocumentSerializer data = DocumentSerializerFactory.create42(new GrowableByteBuffer(64 * 1024, 2.0f)); + DocumentSerializer data = DocumentSerializerFactory.create6(new GrowableByteBuffer(64 * 1024, 2.0f)); data.write(this); return data.getBuf().position(); } @@ -149,7 +149,7 @@ public class Document extends StructuredFieldValue { public final int getApproxSize() { return 4096; } public void serialize(OutputStream out) throws SerializationException { - DocumentSerializer writer = DocumentSerializerFactory.create42(new GrowableByteBuffer(64 * 1024, 2.0f)); + DocumentSerializer writer = DocumentSerializerFactory.create6(new GrowableByteBuffer(64 * 1024, 2.0f)); writer.write(this); GrowableByteBuffer data = writer.getBuf(); byte[] array; @@ -345,38 +345,6 @@ public class Document extends StructuredFieldValue { serialize((DocumentWriter)data); } - @SuppressWarnings("deprecation") - @Deprecated - public void serializeHeader(Serializer data) throws SerializationException { - if (data instanceof DocumentWriter) { - if (data instanceof com.yahoo.document.serialization.VespaDocumentSerializer42) { - ((com.yahoo.document.serialization.VespaDocumentSerializer42)data).setHeaderOnly(true); - } - serialize((DocumentWriter)data); - } else if (data instanceof BufferSerializer) { - serialize(DocumentSerializerFactory.create42(((BufferSerializer) data).getBuf(), true)); - } else { - DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true); - serialize(fw); - data.put(null, fw.getBuf().getByteBuffer()); - } - } - - @Deprecated - public void serializeBody(Serializer data) throws SerializationException { - if (getBody().getFieldCount() > 0) { - if (data instanceof FieldWriter) { - getBody().serialize(new Field("body", getBody().getDataType()), (FieldWriter) data); - } else if (data instanceof BufferSerializer) { - getBody().serialize(new Field("body", getBody().getDataType()), DocumentSerializerFactory.create42(((BufferSerializer) data).getBuf())); - } else { - DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer()); - getBody().serialize(new Field("body", getBody().getDataType()), fw); - data.put(null, fw.getBuf().getByteBuffer()); - } - } - } - @Override public DocumentType getDataType() { return (DocumentType)super.getDataType(); diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java index 5fad35a2287..a3ba27b640f 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManager.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManager.java @@ -266,18 +266,13 @@ public class DocumentTypeManager { } final public Document createDocument(GrowableByteBuffer buf) { - DocumentDeserializer data = DocumentDeserializerFactory.create42(this, buf); + DocumentDeserializer data = DocumentDeserializerFactory.create6(this, buf); return new Document(data); } public Document createDocument(DocumentDeserializer data) { return new Document(data); } - public Document createDocument(GrowableByteBuffer header, GrowableByteBuffer body) { - DocumentDeserializer data = DocumentDeserializerFactory.create42(this, header, body); - return new Document(data); - } - /** * Returns a read only view of the registered data types * diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java index 0bbb57ec60b..ef075662ee7 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUpdate.java +++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java @@ -15,8 +15,10 @@ import com.yahoo.io.GrowableByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; /** @@ -38,7 +40,6 @@ import java.util.Optional; * @see com.yahoo.document.update.FieldUpdate * @see com.yahoo.document.update.ValueUpdate */ -//TODO Vespa 7 Remove all deprecated methods and use a map to avoid quadratic scaling on insert/update/remove public class DocumentUpdate extends DocumentOperation implements Iterable<FieldPathUpdate> { @@ -46,7 +47,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public static final int CLASSID = 0x1000 + 6; private DocumentId docId; - private final List<FieldUpdate> fieldUpdates; + private final Map<Integer, FieldUpdate> id2FieldUpdates; private final List<FieldPathUpdate> fieldPathUpdates; private DocumentType documentType; private Boolean createIfNonExistent; @@ -60,7 +61,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate(DocumentType docType, DocumentId docId) { this.docId = docId; this.documentType = docType; - this.fieldUpdates = new ArrayList<>(); + this.id2FieldUpdates = new HashMap<>(); this.fieldPathUpdates = new ArrayList<>(); } @@ -70,7 +71,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate(DocumentUpdateReader reader) { docId = null; documentType = null; - fieldUpdates = new ArrayList<>(); + id2FieldUpdates = new HashMap<>(); fieldPathUpdates = new ArrayList<>(); reader.read(this); } @@ -113,7 +114,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate applyTo(Document doc) { verifyType(doc); - for (FieldUpdate fieldUpdate : fieldUpdates) { + for (FieldUpdate fieldUpdate : id2FieldUpdates.values()) { fieldUpdate.applyTo(doc); } for (FieldPathUpdate fieldPathUpdate : fieldPathUpdates) { @@ -131,8 +132,9 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP public DocumentUpdate prune(Document doc) { verifyType(doc); - for (Iterator<FieldUpdate> iter = fieldUpdates.iterator(); iter.hasNext();) { - FieldUpdate update = iter.next(); + for (Iterator<Map.Entry<Integer, FieldUpdate>> iter = id2FieldUpdates.entrySet().iterator(); iter.hasNext();) { + Map.Entry<Integer, FieldUpdate> entry = iter.next(); + FieldUpdate update = entry.getValue(); if (!update.isEmpty()) { ValueUpdate last = update.getValueUpdate(update.size() - 1); if (last instanceof AssignValueUpdate) { @@ -158,34 +160,12 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP } /** - * Get an unmodifiable list of all field updates that this document update specifies. - * - * @return a list of all FieldUpdates in this DocumentUpdate - * @deprecated Use fieldUpdates() instead. - */ - @Deprecated - public List<FieldUpdate> getFieldUpdates() { - return Collections.unmodifiableList(fieldUpdates); - } - - /** * Get an unmodifiable collection of all field updates that this document update specifies. * * @return a collection of all FieldUpdates in this DocumentUpdate */ public Collection<FieldUpdate> fieldUpdates() { - return Collections.unmodifiableCollection(fieldUpdates); - } - - /** - * Get an unmodifiable list of all field path updates this document update specifies. - * - * @return Returns a list of all field path updates in this document update. - * @deprecated Use fieldPathUpdates() instead. - */ - @Deprecated - public List<FieldPathUpdate> getFieldPathUpdates() { - return Collections.unmodifiableList(fieldPathUpdates); + return Collections.unmodifiableCollection(id2FieldUpdates.values()); } /** @@ -214,36 +194,6 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP } /** - * Get the field update at the specified index in the list of field updates. - * - * @param index the index of the FieldUpdate to return - * @return the FieldUpdate at the specified index - * @throws IndexOutOfBoundsException if index is out of range - * @deprecated use getFieldUpdate(Field field) instead. - */ - @Deprecated - public FieldUpdate getFieldUpdate(int index) { - return fieldUpdates.get(index); - } - - /** - * Replaces the field update at the specified index in the list of field updates. - * - * @param index index of the FieldUpdate to replace - * @param upd the FieldUpdate to be stored at the specified position - * @return the FieldUpdate previously at the specified position - * @throws IndexOutOfBoundsException if index is out of range - * @deprecated Use removeFieldUpdate/addFieldUpdate instead - */ - @Deprecated - public FieldUpdate setFieldUpdate(int index, FieldUpdate upd) { - FieldUpdate old = fieldUpdates.get(index); - fieldUpdates.set(index, upd); - - return old; - } - - /** * Returns the update for a field * * @param field the field to return the update of @@ -255,7 +205,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP /** Removes all field updates from the list for field updates. */ public void clearFieldUpdates() { - fieldUpdates.clear(); + id2FieldUpdates.clear(); } /** @@ -269,12 +219,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP return field != null ? getFieldUpdate(field) : null; } private FieldUpdate getFieldUpdateById(Integer fieldId) { - for (FieldUpdate fieldUpdate : fieldUpdates) { - if (fieldUpdate.getField().getId() == fieldId) { - return fieldUpdate; - } - } - return null; + return id2FieldUpdates.get(fieldId); } /** @@ -310,7 +255,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return the size of the List of FieldUpdates */ public int size() { - return fieldUpdates.size(); + return id2FieldUpdates.size(); } /** @@ -332,7 +277,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP if (prevUpdate != null) { prevUpdate.addAll(update); } else { - fieldUpdates.add(update); + id2FieldUpdates.put(fieldId, update); } } return this; @@ -372,28 +317,8 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP } } - /** - * Removes the field update at the specified position in the list of field updates. - * - * @param index the index of the FieldUpdate to remove - * @return the FieldUpdate previously at the specified position - * @throws IndexOutOfBoundsException if index is out of range - * @deprecated use removeFieldUpdate(Field field) instead. - */ - @Deprecated - public FieldUpdate removeFieldUpdate(int index) { - return fieldUpdates.remove(index); - } - public FieldUpdate removeFieldUpdate(Field field) { - for (Iterator<FieldUpdate> it = fieldUpdates.iterator(); it.hasNext();) { - FieldUpdate fieldUpdate = it.next(); - if (fieldUpdate.getField().equals(field)) { - it.remove(); - return fieldUpdate; - } - } - return null; + return id2FieldUpdates.remove(field.getId()); } public FieldUpdate removeFieldUpdate(String fieldName) { @@ -411,7 +336,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP } public final void serialize(GrowableByteBuffer buf) { - serialize(DocumentSerializerFactory.create42(buf)); + serialize(DocumentSerializerFactory.create6(buf)); } public void serialize(DocumentUpdateWriter data) { @@ -427,9 +352,8 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP if (docId != null ? !docId.equals(that.docId) : that.docId != null) return false; if (documentType != null ? !documentType.equals(that.documentType) : that.documentType != null) return false; - if (fieldPathUpdates != null ? !fieldPathUpdates.equals(that.fieldPathUpdates) : that.fieldPathUpdates != null) - return false; - if (fieldUpdates != null ? !fieldUpdates.equals(that.fieldUpdates) : that.fieldUpdates != null) return false; + if (!fieldPathUpdates.equals(that.fieldPathUpdates)) return false; + if (!id2FieldUpdates.equals(that.id2FieldUpdates)) return false; if (this.getCreateIfNonExistent() != ((DocumentUpdate) o).getCreateIfNonExistent()) return false; return true; @@ -438,8 +362,8 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP @Override public int hashCode() { int result = docId != null ? docId.hashCode() : 0; - result = 31 * result + (fieldUpdates != null ? fieldUpdates.hashCode() : 0); - result = 31 * result + (fieldPathUpdates != null ? fieldPathUpdates.hashCode() : 0); + result = 31 * result + id2FieldUpdates.hashCode(); + result = 31 * result + fieldPathUpdates.hashCode(); result = 31 * result + (documentType != null ? documentType.hashCode() : 0); return result; } @@ -455,7 +379,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP string.append(": "); string.append("["); - for (FieldUpdate fieldUpdate : fieldUpdates) { + for (FieldUpdate fieldUpdate : id2FieldUpdates.values()) { string.append(fieldUpdate).append(" "); } string.append("]"); @@ -482,14 +406,13 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP * @return True if this update is empty. */ public boolean isEmpty() { - return fieldUpdates.isEmpty() && fieldPathUpdates.isEmpty(); + return id2FieldUpdates.isEmpty() && fieldPathUpdates.isEmpty(); } /** * Sets whether this update should create the document it updates if that document does not exist. * In this case an empty document is created before the update is applied. * - * @since 5.17 * @param value Whether the document it updates should be created. */ public void setCreateIfNonExistent(boolean value) { @@ -499,8 +422,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP /** * Gets whether this update should create the document it updates if that document does not exist. * - * @since 5.17 - * @return Whether the document it updates should be created. + * @return whether the document it updates should be created. */ public boolean getCreateIfNonExistent() { return createIfNonExistent != null && createIfNonExistent; diff --git a/document/src/main/java/com/yahoo/document/Field.java b/document/src/main/java/com/yahoo/document/Field.java index 5d3d148d832..d0a19a50073 100644 --- a/document/src/main/java/com/yahoo/document/Field.java +++ b/document/src/main/java/com/yahoo/document/Field.java @@ -187,7 +187,7 @@ public class Field extends FieldBase implements FieldSet, Comparable, Serializab * @param type The new type of the field. * @deprecated do not use */ - @Deprecated // TODO: Remove on Vespa 8 + @Deprecated // todo - refactor SD processing to avoid needing this public void setDataType(DataType type) { dataType = type; fieldId = calculateIdV7(null); diff --git a/document/src/main/java/com/yahoo/document/PositionDataType.java b/document/src/main/java/com/yahoo/document/PositionDataType.java index bb110ee7219..0e69171c106 100644 --- a/document/src/main/java/com/yahoo/document/PositionDataType.java +++ b/document/src/main/java/com/yahoo/document/PositionDataType.java @@ -100,12 +100,12 @@ public final class PositionDataType { } public static String getPositionSummaryFieldName(String fieldName) { - // TODO for 6.0, rename to _position to use a field name that is actually legal + // TODO for Vespa 8, consider renaming to _position to use a field name that is actually legal return fieldName + ".position"; } public static String getDistanceSummaryFieldName(String fieldName) { - // TODO for 6.0, rename to _distance to use a field name that is actually legal + // TODO for Vespa 8, consider renaming to _distance to use a field name that is actually legal return fieldName + ".distance"; } diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java index dc3fd36b367..f895ad21b0a 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java @@ -52,7 +52,7 @@ public abstract class FieldValue extends Identifiable implements Comparable<Fiel } final public void serialize(GrowableByteBuffer buf) { - serialize(DocumentSerializerFactory.create42(buf)); + serialize(DocumentSerializerFactory.create6(buf)); } public abstract void printXml(XmlStream xml); @@ -140,9 +140,9 @@ public abstract class FieldValue extends Identifiable implements Comparable<Fiel if (target instanceof FieldWriter) { serialize(null, (FieldWriter) target); } else if (target instanceof BufferSerializer) { - serialize(null, DocumentSerializerFactory.create42(((BufferSerializer) target).getBuf())); + serialize(null, DocumentSerializerFactory.create6(((BufferSerializer) target).getBuf())); } else { - DocumentSerializer fw = DocumentSerializerFactory.create42(new GrowableByteBuffer()); + DocumentSerializer fw = DocumentSerializerFactory.create6(new GrowableByteBuffer()); serialize(null, fw); target.put(null, fw.getBuf().getByteBuffer()); } diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java index 9fac025c15e..74b94b8135e 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/AddFieldPathUpdate.java @@ -8,7 +8,7 @@ import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.FieldPathIteratorHandler; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.serialization.DocumentUpdateReader; -import com.yahoo.document.serialization.VespaDocumentSerializerHead; +import com.yahoo.document.serialization.VespaDocumentSerializer6; /** * @author <a href="mailto:thomasg@yahoo-inc.com">Thomas Gundersen</a> @@ -96,7 +96,7 @@ public class AddFieldPathUpdate extends FieldPathUpdate { } @Override - public void serialize(VespaDocumentSerializerHead data) { + public void serialize(VespaDocumentSerializer6 data) { data.write(this); } diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java index e31eefa2c00..b01742018a2 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/AssignFieldPathUpdate.java @@ -9,7 +9,7 @@ import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.NumericFieldValue; import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.serialization.DocumentUpdateReader; -import com.yahoo.document.serialization.VespaDocumentSerializerHead; +import com.yahoo.document.serialization.VespaDocumentSerializer6; import java.util.HashMap; import java.util.Map; @@ -237,7 +237,7 @@ public class AssignFieldPathUpdate extends FieldPathUpdate { } @Override - public void serialize(VespaDocumentSerializerHead data) { + public void serialize(VespaDocumentSerializer6 data) { data.write(this); } diff --git a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java index 29c876b85fe..d4144116a03 100644 --- a/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java +++ b/document/src/main/java/com/yahoo/document/fieldpathupdate/FieldPathUpdate.java @@ -11,7 +11,7 @@ import com.yahoo.document.select.Result; import com.yahoo.document.select.ResultList; import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.serialization.DocumentUpdateReader; -import com.yahoo.document.serialization.VespaDocumentSerializerHead; +import com.yahoo.document.serialization.VespaDocumentSerializer6; import java.util.ListIterator; /** @@ -130,7 +130,7 @@ public abstract class FieldPathUpdate { } } - public void serialize(VespaDocumentSerializerHead data) { + public void serialize(VespaDocumentSerializer6 data) { data.write(this); } diff --git a/document/src/main/java/com/yahoo/document/select/package-info.java b/document/src/main/java/com/yahoo/document/select/package-info.java index e6de6497008..813643a7f06 100644 --- a/document/src/main/java/com/yahoo/document/select/package-info.java +++ b/document/src/main/java/com/yahoo/document/select/package-info.java @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -@PublicApi package com.yahoo.document.select; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java index 072f11f5f22..03aa409ee8d 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializerFactory.java @@ -13,25 +13,26 @@ public class DocumentDeserializerFactory { /** * Creates a de-serializer for the current head document format. - * This format is an extension of the 4.2 format. + * This format is an extension of the 6.x format. */ public static DocumentDeserializer createHead(DocumentTypeManager manager, GrowableByteBuffer buf) { return new VespaDocumentDeserializerHead(manager, buf); } /** - * Creates a de-serializer for the document format that was created on Vespa 4.2. + * Creates a de-serializer for the 6.x document format. + * This format is an extension of the 4.2 format. */ - @SuppressWarnings("deprecation") - public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf) { - return new VespaDocumentDeserializer42(manager, buf); + public static DocumentDeserializer create6(DocumentTypeManager manager, GrowableByteBuffer buf) { + return new VespaDocumentDeserializer6(manager, buf); } /** * Creates a de-serializer for the document format that was created on Vespa 4.2. */ @SuppressWarnings("deprecation") - public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf, GrowableByteBuffer body) { - return new VespaDocumentDeserializer42(manager, buf, body); + public static DocumentDeserializer create42(DocumentTypeManager manager, GrowableByteBuffer buf) { + return new VespaDocumentDeserializer42(manager, buf); } + } diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java index 7ff58855c34..54ec4e2fcca 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java @@ -12,26 +12,34 @@ public class DocumentSerializerFactory { /** * Creates a serializer for the current head document format. - * This format is an extension of the 4.2 format. + * This format is an extension of the 6.x format. */ public static DocumentSerializer createHead(GrowableByteBuffer buf) { return new VespaDocumentSerializerHead(buf); } /** - * Creates a serializer for the document format that was created on Vespa 4.2. + * Creates a serializer for the 6.x document format. + * This format is an extension of the 4.2 format. */ - @SuppressWarnings("deprecation") - public static DocumentSerializer create42(GrowableByteBuffer buf) { - return new VespaDocumentSerializer42(buf); + public static DocumentSerializer create6(GrowableByteBuffer buf) { + return new VespaDocumentSerializer6(buf); + } + + /** + * Creates a serializer for the 6.x document format. + * This format is an extension of the 4.2 format. + */ + public static DocumentSerializer create6() { + return new VespaDocumentSerializer6(new GrowableByteBuffer()); } /** * Creates a serializer for the document format that was created on Vespa 4.2. */ @SuppressWarnings("deprecation") - public static DocumentSerializer create42(GrowableByteBuffer buf, boolean headerOnly) { - return new VespaDocumentSerializer42(buf, headerOnly); + public static DocumentSerializer create42(GrowableByteBuffer buf) { + return new VespaDocumentSerializer42(buf); } /** diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java index 7ff5729ca39..7ec4433a24f 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer42.java @@ -63,6 +63,7 @@ import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; import com.yahoo.text.Utf8Array; import com.yahoo.text.Utf8String; +import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.FieldBase; import java.nio.ByteBuffer; @@ -80,9 +81,9 @@ import static com.yahoo.text.Utf8.calculateStringPositions; * @deprecated Please use {@link com.yahoo.document.serialization.VespaDocumentDeserializerHead} instead for new code. * @author baldersheim */ -@Deprecated // Remove on Vespa 7 +@Deprecated // TODO: Remove on Vespa 8 // When removing: Move content of this class into VespaDocumentDeserializerHead -public class VespaDocumentDeserializer42 extends VespaDocumentSerializer42 implements DocumentDeserializer { +public class VespaDocumentDeserializer42 extends BufferSerializer implements DocumentDeserializer { private final Compressor compressor = new Compressor(); private DocumentTypeManager manager; diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java new file mode 100644 index 00000000000..5424798110c --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -0,0 +1,880 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.collections.Tuple2; +import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; +import com.yahoo.document.annotation.AlternateSpanList; +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.annotation.Span; +import com.yahoo.document.annotation.SpanList; +import com.yahoo.document.annotation.SpanNode; +import com.yahoo.document.annotation.SpanNodeParent; +import com.yahoo.document.annotation.SpanTree; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.ReferenceFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; +import com.yahoo.document.MapDataType; +import com.yahoo.document.predicate.BinaryFormat; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.StructDataType; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.ArithmeticValueUpdate; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.ClearValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.RemoveValueUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8Array; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.objects.BufferSerializer; +import com.yahoo.vespa.objects.FieldBase; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.text.Utf8.calculateStringPositions; + +/** + * Class used for de-serializing documents on the Vespa 6.x document format. + * + * @author baldersheim + */ +public class VespaDocumentDeserializer6 extends BufferSerializer implements DocumentDeserializer { + + private final Compressor compressor = new Compressor(); + private DocumentTypeManager manager; + private short version; + private List<SpanNode> spanNodes; + private List<Annotation> annotations; + private int[] stringPositions; + + VespaDocumentDeserializer6(DocumentTypeManager manager, GrowableByteBuffer buf) { + super(buf); + this.manager = manager; + this.version = Document.SERIALIZED_VERSION; + } + + final public DocumentTypeManager getDocumentTypeManager() { return manager; } + + public void read(Document document) { + read(null, document); + } + + @SuppressWarnings("deprecation") + public void read(FieldBase field, Document doc) { + // Verify that we have correct version + version = getShort(null); + if (version < 6 || version > Document.SERIALIZED_VERSION) { + throw new DeserializationException("Unknown version " + version + ", expected " + + Document.SERIALIZED_VERSION + "."); + } + + int dataLength = 0; + int dataPos = 0; + + if (version < 7) { + getInt2_4_8Bytes(null); // Total document size.. Ignore + } else { + dataLength = getInt(null); + dataPos = position(); + } + + doc.setId(readDocumentId()); + + Byte content = getByte(null); + + doc.setDataType(readDocumentType()); + + Struct h = doc.getHeader(); + Struct b = doc.getBody(); + h.clear(); + b.clear(); + if ((content & 0x2) != 0) { + readHeaderBody(h, b); + } + if ((content & 0x4) != 0) { + readHeaderBody(b, h); + } + + if (version < 8) { + int crcVal = getInt(null); + } + + if (version > 6) { + if (dataLength != (position() - dataPos)) { + throw new DeserializationException("Length mismatch"); + } + } + } + public void read(FieldBase field, FieldValue value) { + throw new IllegalArgumentException("read not implemented yet."); + } + + public <T extends FieldValue> void read(FieldBase field, Array<T> array) { + int numElements = getNumCollectionElems(); + ArrayList<T> list = new ArrayList<T>(numElements); + ArrayDataType type = array.getDataType(); + for (int i = 0; i < numElements; i++) { + if (version < 7) { + getInt(null); // We don't need size for anything + } + FieldValue fv = type.getNestedType().createFieldValue(); + fv.deserialize(null, this); + list.add((T) fv); + } + array.clear(); + array.addAll(list); + } + + public <K extends FieldValue, V extends FieldValue> void read(FieldBase field, MapFieldValue<K, V> map) { + int numElements = getNumCollectionElems(); + Map<K,V> hash = new HashMap<>(); + MapDataType type = map.getDataType(); + for (int i = 0; i < numElements; i++) { + if (version < 7) { + getInt(null); // We don't need size for anything + } + K key = (K) type.getKeyType().createFieldValue(); + V val = (V) type.getValueType().createFieldValue(); + key.deserialize(null, this); + val.deserialize(null, this); + hash.put(key, val); + } + map.clear(); + map.putAll(hash); + } + + private int getNumCollectionElems() { + int numElements; + if (version < 7) { + getInt(null); // We already know the nested type, so ignore that.. + numElements = getInt(null); + } else { + numElements = getInt1_2_4Bytes(null); + } + if (numElements < 0) { + throw new DeserializationException("Bad number of array/map elements, " + numElements); + } + return numElements; + } + + public <T extends FieldValue> void read(FieldBase field, CollectionFieldValue<T> value) { + throw new IllegalArgumentException("read not implemented yet."); + } + public void read(FieldBase field, ByteFieldValue value) { value.assign(getByte(null)); } + + @Override + public void read(FieldBase field, BoolFieldValue value) { + value.setBoolean((getByte(null) != 0)); + } + + public void read(FieldBase field, DoubleFieldValue value) { value.assign(getDouble(null)); } + public void read(FieldBase field, FloatFieldValue value) { value.assign(getFloat(null)); } + public void read(FieldBase field, IntegerFieldValue value) { value.assign(getInt(null)); } + public void read(FieldBase field, LongFieldValue value) { value.assign(getLong(null)); } + + public void read(FieldBase field, Raw value) { + int rawsize = getInt(null); + byte[] rawBytes = getBytes(null, rawsize); + value.assign(rawBytes); + } + + @Override + public void read(FieldBase field, PredicateFieldValue value) { + int len = getInt(null); + byte[] buf = getBytes(null, len); + value.assign(BinaryFormat.decode(buf)); + } + + public void read(FieldBase field, StringFieldValue value) { + byte coding = getByte(null); + + int length = getInt1_4Bytes(null); + + //OK, it seems that this length includes null termination. + //NOTE: the following four lines are basically parseNullTerminatedString() inlined, + //but we need to use the UTF-8 buffer below, so not using that method... + byte[] stringArray = new byte[length - 1]; + buf.get(stringArray); + buf.get(); //move past 0-termination + value.setUnChecked(Utf8.toString(stringArray)); + + if ((coding & 64) == 64) { + //we have a span tree! + try { + //we don't support serialization of nested span trees, so this is safe: + stringPositions = calculateStringPositions(stringArray); + //total length: + int size = buf.getInt(); + int startPos = buf.position(); + + int numSpanTrees = buf.getInt1_2_4Bytes(); + + for (int i = 0; i < numSpanTrees; i++) { + SpanTree tree = new SpanTree(); + StringFieldValue treeName = new StringFieldValue(); + treeName.deserialize(this); + tree.setName(treeName.getString()); + value.setSpanTree(tree); + readSpanTree(tree, false); + } + + buf.position(startPos + size); + } finally { + stringPositions = null; + } + } + } + + @Override + public void read(FieldBase field, TensorFieldValue value) { + int encodedTensorLength = buf.getInt1_4Bytes(); + if (encodedTensorLength > 0) { + byte[] encodedTensor = getBytes(null, encodedTensorLength); + value.assign(TypedBinaryFormat.decode(Optional.of(value.getDataType().getTensorType()), + GrowableByteBuffer.wrap(encodedTensor))); + } else { + value.clear(); + } + } + + @Override + public void read(FieldBase field, ReferenceFieldValue value) { + final boolean documentIdPresent = (buf.get() != 0); + if (documentIdPresent) { + value.assign(readDocumentId()); + } else { + value.clear(); + } + } + + public void read(FieldBase fieldDef, Struct s) { + s.setVersion(version); + int startPos = position(); + + if (version < 6) { + throw new DeserializationException("Illegal document serialization version " + version); + } + + int dataSize; + if (version < 7) { + long rSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (rSize > Integer.MAX_VALUE) { + throw new DeserializationException("Raw size of data block is too large."); + } + dataSize = (int)rSize; + } else { + dataSize = getInt(null); + } + + byte comprCode = getByte(null); + CompressionType compression = CompressionType.valueOf(comprCode); + + int uncompressedSize = 0; + if (compression != CompressionType.NONE && + compression != CompressionType.INCOMPRESSIBLE) + { + // uncompressedsize (full size of FIELDS only, after decompression) + long pSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (pSize > Integer.MAX_VALUE) { + throw new DeserializationException("Uncompressed size of data block is too large."); + } + uncompressedSize = (int) pSize; + } + + int numberOfFields = getInt1_4Bytes(null); + + List<Tuple2<Integer, Long>> fieldIdsAndLengths = new ArrayList<>(numberOfFields); + for (int i=0; i<numberOfFields; ++i) { + // id, length (length only used for unknown fields + fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null))); + } + + // save a reference to the big buffer we're reading from: + GrowableByteBuffer bigBuf = buf; + + if (version < 7) { + // In V6 and earlier, the length included the header. + int headerSize = position() - startPos; + dataSize -= headerSize; + } + byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize)); + + // set position in original buffer to after data + position(position() + dataSize); + + // for a while: deserialize from this buffer instead: + buf = GrowableByteBuffer.wrap(destination); + + s.clear(); + StructDataType type = s.getDataType(); + for (int i=0; i<numberOfFields; ++i) { + Field structField = type.getField(fieldIdsAndLengths.get(i).first, version); + if (structField == null) { + //ignoring unknown field: + position(position() + fieldIdsAndLengths.get(i).second.intValue()); + } else { + int posBefore = position(); + FieldValue value = structField.getDataType().createFieldValue(); + value.deserialize(structField, this); + s.setFieldValue(structField, value); + //jump to beginning of next field: + position(posBefore + fieldIdsAndLengths.get(i).second.intValue()); + } + } + + // restore the original buffer + buf = bigBuf; + } + + private void readHeaderBody(Struct primary, Struct alternate) { + primary.setVersion(version); + int startPos = position(); + + if (version < 6) { + throw new DeserializationException("Illegal document serialization version " + version); + } + + int dataSize; + if (version < 7) { + long rSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (rSize > Integer.MAX_VALUE) { + throw new DeserializationException("Raw size of data block is too large."); + } + dataSize = (int)rSize; + } else { + dataSize = getInt(null); + } + + byte comprCode = getByte(null); + CompressionType compression = CompressionType.valueOf(comprCode); + + int uncompressedSize = 0; + if (compression != CompressionType.NONE && + compression != CompressionType.INCOMPRESSIBLE) + { + // uncompressedsize (full size of FIELDS only, after decompression) + long pSize = getInt2_4_8Bytes(null); + //TODO: Look into how to support data segments larger than INT_MAX bytes + if (pSize > Integer.MAX_VALUE) { + throw new DeserializationException("Uncompressed size of data block is too large."); + } + uncompressedSize = (int) pSize; + } + + int numberOfFields = getInt1_4Bytes(null); + + List<Tuple2<Integer, Long>> fieldIdsAndLengths = new ArrayList<>(numberOfFields); + for (int i=0; i<numberOfFields; ++i) { + // id, length (length only used for unknown fields + fieldIdsAndLengths.add(new Tuple2<>(getInt1_4Bytes(null), getInt2_4_8Bytes(null))); + } + + // save a reference to the big buffer we're reading from: + GrowableByteBuffer bigBuf = buf; + + if (version < 7) { + // In V6 and earlier, the length included the header. + int headerSize = position() - startPos; + dataSize -= headerSize; + } + byte[] destination = compressor.decompress(compression, getBuf().array(), position(), uncompressedSize, Optional.of(dataSize)); + + // set position in original buffer to after data + position(position() + dataSize); + + // for a while: deserialize from this buffer instead: + buf = GrowableByteBuffer.wrap(destination); + + StructDataType priType = primary.getDataType(); + StructDataType altType = alternate.getDataType(); + for (int i=0; i<numberOfFields; ++i) { + int posBefore = position(); + Struct s = null; + Integer f_id = fieldIdsAndLengths.get(i).first; + Field structField = priType.getField(f_id, version); + if (structField != null) { + s = primary; + } else { + structField = altType.getField(f_id, version); + if (structField != null) { + s = alternate; + } + } + if (s != null) { + FieldValue value = structField.getDataType().createFieldValue(); + value.deserialize(structField, this); + s.setFieldValue(structField, value); + } + //jump to beginning of next field: + position(posBefore + fieldIdsAndLengths.get(i).second.intValue()); + } + + // restore the original buffer + buf = bigBuf; + } + + public void read(FieldBase field, StructuredFieldValue value) { + throw new IllegalArgumentException("read not implemented yet."); + } + public <T extends FieldValue> void read(FieldBase field, WeightedSet<T> ws) { + WeightedSetDataType type = ws.getDataType(); + getInt(null); // Have no need for type + + int numElements = getInt(null); + if (numElements < 0) { + throw new DeserializationException("Bad number of weighted set elements, " + numElements); + } + + ws.clearAndReserve(numElements * 2); // Avoid resizing + for (int i = 0; i < numElements; i++) { + int size = getInt(null); + FieldValue value = type.getNestedType().createFieldValue(); + value.deserialize(null, this); + IntegerFieldValue weight = new IntegerFieldValue(getInt(null)); + ws.putUnChecked((T) value, weight); + } + + } + + public void read(FieldBase field, AnnotationReference value) { + int seqId = buf.getInt1_2_4Bytes(); + try { + Annotation a = annotations.get(seqId); + value.setReferenceNoCompatibilityCheck(a); + } catch (IndexOutOfBoundsException iiobe) { + throw new SerializationException("Could not serialize AnnotationReference value, reference not found.", iiobe); + } + } + + private Utf8String deserializeAttributeString() throws DeserializationException { + int length = getByte(null); + return new Utf8String(parseNullTerminatedString(length)); + } + + private Utf8Array parseNullTerminatedString() { return parseNullTerminatedString(getBuf().getByteBuffer()); } + private Utf8Array parseNullTerminatedString(int lengthExcludingNull) { return parseNullTerminatedString(getBuf().getByteBuffer(), lengthExcludingNull); } + + static Utf8Array parseNullTerminatedString(ByteBuffer buf, int lengthExcludingNull) throws DeserializationException { + Utf8Array utf8 = new Utf8Array(buf, lengthExcludingNull); + buf.get(); //move past 0-termination + return utf8; + } + + static Utf8Array parseNullTerminatedString(ByteBuffer buf) throws DeserializationException { + //search for 0-byte + int end = getFirstNullByte(buf); + + if (end == -1) { + throw new DeserializationException("Could not locate terminating 0-byte for string"); + } + + return parseNullTerminatedString(buf, end - buf.position()); + } + + private static int getFirstNullByte(ByteBuffer buf) { + int end = -1; + int start = buf.position(); + + while (true) { + try { + byte dataByte = buf.get(); + if (dataByte == (byte) 0) { + end = buf.position() - 1; + break; + } + } catch (Exception e) { + break; + } + } + + buf.position(start); + return end; + } + + public void read(DocumentUpdate update) { + update.setId(new DocumentId(this)); + update.setDocumentType(readDocumentType()); + + int size = getInt(null); + + for (int i = 0; i < size; i++) { + update.addFieldUpdate(new FieldUpdate(this, update.getDocumentType(), 8)); + } + + int sizeAndFlags = getInt(null); + update.setCreateIfNonExistent(DocumentUpdateFlags.extractFlags(sizeAndFlags).getCreateIfNonExistent()); + size = DocumentUpdateFlags.extractValue(sizeAndFlags); + + for (int i = 0; i < size; i++) { + int type = getByte(null); + update.addFieldPathUpdate(FieldPathUpdate.create(FieldPathUpdate.Type.valueOf(type), + update.getDocumentType(), this)); + } + } + + + public void read(FieldPathUpdate update) { + String fieldPath = getString(null); + String whereClause = getString(null); + update.setFieldPath(fieldPath); + + try { + update.setWhereClause(whereClause); + } catch (ParseException e) { + throw new DeserializationException(e); + } + } + + public void read(AssignFieldPathUpdate update) { + byte flags = getByte(null); + update.setRemoveIfZero((flags & AssignFieldPathUpdate.REMOVE_IF_ZERO) != 0); + update.setCreateMissingPath((flags & AssignFieldPathUpdate.CREATE_MISSING_PATH) != 0); + if ((flags & AssignFieldPathUpdate.ARITHMETIC_EXPRESSION) != 0) { + update.setExpression(getString(null)); + } else { + DataType dt = update.getFieldPath().getResultingDataType(); + FieldValue fv = dt.createFieldValue(); + fv.deserialize(this); + update.setNewValue(fv); + } + } + + public void read(RemoveFieldPathUpdate update) { + + } + + public void read(AddFieldPathUpdate update) { + DataType dt = update.getFieldPath().getResultingDataType(); + FieldValue fv = dt.createFieldValue(); + dt.createFieldValue(); + fv.deserialize(this); + + if (!(fv instanceof Array)) { + throw new DeserializationException("Add only applicable to array types"); + } + update.setNewValues((Array)fv); + } + + public ValueUpdate getValueUpdate(DataType superType, DataType subType) { + int vuTypeId = getInt(null); + + ValueUpdate.ValueUpdateClassID op = ValueUpdate.ValueUpdateClassID.getID(vuTypeId); + if (op == null) { + throw new IllegalArgumentException("Read type "+vuTypeId+" of bytebuffer, but this is not a legal value update type."); + } + + switch (op) { + case ADD: + { + FieldValue fval = subType.createFieldValue(); + fval.deserialize(this); + int weight = getInt(null); + return new AddValueUpdate(fval, weight); + } + case ARITHMETIC: + int opId = getInt(null); + ArithmeticValueUpdate.Operator operator = ArithmeticValueUpdate.Operator.getID(opId); + double operand = getDouble(null); + return new ArithmeticValueUpdate(operator, operand); + case ASSIGN: + { + byte contents = getByte(null); + FieldValue fval = null; + if (contents == (byte) 1) { + fval = superType.createFieldValue(); + fval.deserialize(this); + } + return new AssignValueUpdate(fval); + } + case CLEAR: + return new ClearValueUpdate(); + case MAP: + if (superType instanceof ArrayDataType) { + CollectionDataType type = (CollectionDataType) superType; + IntegerFieldValue index = new IntegerFieldValue(); + index.deserialize(this); + ValueUpdate update = getValueUpdate(type.getNestedType(), null); + return new MapValueUpdate(index, update); + } else if (superType instanceof WeightedSetDataType) { + CollectionDataType type = (CollectionDataType) superType; + FieldValue fval = type.getNestedType().createFieldValue(); + fval.deserialize(this); + ValueUpdate update = getValueUpdate(DataType.INT, null); + return new MapValueUpdate(fval, update); + } else { + throw new DeserializationException("MapValueUpdate only works for arrays and weighted sets"); + } + case REMOVE: + FieldValue fval = ((CollectionDataType) superType).getNestedType().createFieldValue(); + fval.deserialize(this); + return new RemoveValueUpdate(fval); + default: + throw new DeserializationException( + "Could not deserialize ValueUpdate, unknown valueUpdateClassID type " + vuTypeId); + } + } + + public void read(FieldUpdate fieldUpdate) { + int fieldId = getInt(null); + Field field = fieldUpdate.getDocumentType().getField(fieldId, fieldUpdate.getSerializationVersion()); + if (field == null) { + throw new DeserializationException( + "Cannot deserialize FieldUpdate, field fieldId " + fieldId + " not found in " + fieldUpdate.getDocumentType()); + } + + fieldUpdate.setField(field); + int size = getInt(null); + + for (int i = 0; i < size; i++) { + if (field.getDataType() instanceof CollectionDataType) { + CollectionDataType collType = (CollectionDataType) field.getDataType(); + fieldUpdate.addValueUpdate(getValueUpdate(collType, collType.getNestedType())); + } else { + fieldUpdate.addValueUpdate(getValueUpdate(field.getDataType(), null)); + } + } + } + + public DocumentId readDocumentId() { + Utf8String uri = new Utf8String(parseNullTerminatedString(getBuf().getByteBuffer())); + return DocumentId.createFromSerialized(uri.toString()); + } + + public DocumentType readDocumentType() { + Utf8Array docTypeName = parseNullTerminatedString(); + int ignored = getShort(null); // used to hold the version + + DocumentType docType = manager.getDocumentType(new DataTypeName(docTypeName)); + if (docType == null) { + throw new DeserializationException("No known document type with name " + + new Utf8String(docTypeName).toString()); + } + return docType; + } + + private SpanNode readSpanNode() { + byte type = buf.get(); + buf.position(buf.position() - 1); + + SpanNode retval; + if ((type & Span.ID) == Span.ID) { + retval = new Span(); + if (spanNodes != null) { + spanNodes.add(retval); + } + read((Span) retval); + } else if ((type & SpanList.ID) == SpanList.ID) { + retval = new SpanList(); + if (spanNodes != null) { + spanNodes.add(retval); + } + read((SpanList) retval); + } else if ((type & AlternateSpanList.ID) == AlternateSpanList.ID) { + retval = new AlternateSpanList(); + if (spanNodes != null) { + spanNodes.add(retval); + } + read((AlternateSpanList) retval); + } else { + throw new DeserializationException("Cannot read SpanNode of type " + type); + } + return retval; + } + + private void readSpanTree(SpanTree tree, boolean readName) { + //we don't support serialization of nested span trees: + if (spanNodes != null || annotations != null) { + throw new SerializationException("Deserialization of nested SpanTrees is not supported."); + } + + //we're going to write a new SpanTree, create a new Map for nodes: + spanNodes = new ArrayList<SpanNode>(); + annotations = new ArrayList<Annotation>(); + + try { + if (readName) { + StringFieldValue treeName = new StringFieldValue(); + treeName.deserialize(this); + tree.setName(treeName.getString()); + } + + SpanNode root = readSpanNode(); + tree.setRoot(root); + + int numAnnotations = buf.getInt1_2_4Bytes(); + + for (int i = 0; i < numAnnotations; i++) { + Annotation a = new Annotation(); + annotations.add(a); + } + for (int i = 0; i < numAnnotations; i++) { + read(annotations.get(i)); + } + for (Annotation a : annotations) { + tree.annotate(a); + } + + for (SpanNode node: spanNodes) { + if (node instanceof Span) { + correctIndexes((Span) node); + } + } + } finally { + //we're done, let's set this to null to save memory and prevent madness: + spanNodes = null; + annotations = null; + } + } + + public void read(SpanTree tree) { + readSpanTree(tree, true); + } + + public void read(Annotation annotation) { + int annotationTypeId = buf.getInt(); + AnnotationType type = manager.getAnnotationTypeRegistry().getType(annotationTypeId); + + if (type == null) { + throw new DeserializationException("Cannot deserialize annotation of type " + annotationTypeId + " (unknown type)"); + } + + annotation.setType(type); + + byte features = buf.get(); + int length = buf.getInt1_2_4Bytes(); + + if ((features & (byte) 1) == (byte) 1) { + //we have a span node + int spanNodeId = buf.getInt1_2_4Bytes(); + try { + SpanNode node = spanNodes.get(spanNodeId); + annotation.setSpanNode(node); + } catch (IndexOutOfBoundsException ioobe) { + throw new DeserializationException("Could not deserialize annotation, associated span node not found ", ioobe); + } + } + if ((features & (byte) 2) == (byte) 2) { + //we have a value: + int dataTypeId = buf.getInt(); + + //if this data type ID the same as the one in our config? + if (dataTypeId != type.getDataType().getId()) { + //not the same, but we will handle it gracefully, and just skip past the data: + buf.position(buf.position() + length - 4); + } else { + FieldValue value = type.getDataType().createFieldValue(); + value.deserialize(this); + annotation.setFieldValue(value); + } + } + } + + public void read(Span span) { + byte type = buf.get(); + if ((type & Span.ID) != Span.ID) { + throw new DeserializationException("Cannot deserialize Span with type " + type); + } + span.setFrom(buf.getInt1_2_4Bytes()); + span.setLength(buf.getInt1_2_4Bytes()); + } + + private void correctIndexes(Span span) { + if (stringPositions == null) { + throw new DeserializationException("Cannot deserialize Span, no access to parent StringFieldValue."); + } + int fromIndex = stringPositions[span.getFrom()]; + int toIndex = stringPositions[span.getTo()]; + int length = toIndex - fromIndex; + + span.setFrom(fromIndex); + span.setLength(length); + } + + public void read(SpanList spanList) { + byte type = buf.get(); + if ((type & SpanList.ID) != SpanList.ID) { + throw new DeserializationException("Cannot deserialize SpanList with type " + type); + } + List<SpanNode> nodes = readSpanList(spanList); + for (SpanNode node : nodes) { + spanList.add(node); + } + } + + public void read(AlternateSpanList altSpanList) { + byte type = buf.get(); + if ((type & AlternateSpanList.ID) != AlternateSpanList.ID) { + throw new DeserializationException("Cannot deserialize AlternateSpanList with type " + type); + } + int numSubTrees = buf.getInt1_2_4Bytes(); + + for (int i = 0; i < numSubTrees; i++) { + double prob = buf.getDouble(); + List<SpanNode> list = readSpanList(altSpanList); + + if (i == 0) { + for (SpanNode node : list) { + altSpanList.add(node); + } + altSpanList.setProbability(0, prob); + } else { + altSpanList.addChildren(i, list, prob); + } + } + } + + private List<SpanNode> readSpanList(SpanNodeParent parent) { + int size = buf.getInt1_2_4Bytes(); + List<SpanNode> spanList = new ArrayList<SpanNode>(); + for (int i = 0; i < size; i++) { + spanList.add(readSpanNode()); + } + return spanList; + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java index 40aec94aec6..1e90395a153 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java @@ -1,14 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; -import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.datatypes.BoolFieldValue; -import com.yahoo.document.fieldpathupdate.FieldPathUpdate; -import com.yahoo.document.update.FieldUpdate; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.vespa.objects.FieldBase; /** * Class used for de-serializing documents on the current head document format. @@ -16,36 +10,10 @@ import com.yahoo.vespa.objects.FieldBase; * @author baldersheim */ @SuppressWarnings("deprecation") -public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer42 { +public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer6 { public VespaDocumentDeserializerHead(DocumentTypeManager manager, GrowableByteBuffer buffer) { super(manager, buffer); } - @Override - public void read(DocumentUpdate update) { - update.setId(new DocumentId(this)); - update.setDocumentType(readDocumentType()); - - int size = getInt(null); - - for (int i = 0; i < size; i++) { - update.addFieldUpdate(new FieldUpdate(this, update.getDocumentType(), 8)); - } - - int sizeAndFlags = getInt(null); - update.setCreateIfNonExistent(DocumentUpdateFlags.extractFlags(sizeAndFlags).getCreateIfNonExistent()); - size = DocumentUpdateFlags.extractValue(sizeAndFlags); - - for (int i = 0; i < size; i++) { - int type = getByte(null); - update.addFieldPathUpdate(FieldPathUpdate.create(FieldPathUpdate.Type.valueOf(type), - update.getDocumentType(), this)); - } - } - - @Override - public void read(FieldBase field, BoolFieldValue value) { - value.setBoolean((getByte(null) != 0)); - } } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java index 581c7df8aee..ebe9a124033 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer42.java @@ -67,11 +67,10 @@ import static com.yahoo.text.Utf8.calculateBytePositions; * @deprecated use {@link com.yahoo.document.serialization.VespaDocumentSerializerHead} instead for new code * @author baldersheim */ -@Deprecated // OK: Don't remove on Vespa 6: Mail may have documents on this format still +@Deprecated // TODO: Remove on Vespa 8 // When removing: Move content into VespaDocumentSerializerHead public class VespaDocumentSerializer42 extends BufferSerializer implements DocumentSerializer { - private boolean headerOnly; private int spanNodeCounter = -1; private int[] bytePositions; @@ -83,55 +82,45 @@ public class VespaDocumentSerializer42 extends BufferSerializer implements Docum super(); } - VespaDocumentSerializer42(GrowableByteBuffer buf, boolean headerOnly) { - this(buf); - this.headerOnly = headerOnly; - } - - public void setHeaderOnly(boolean headerOnly) { - this.headerOnly = headerOnly; - } - public void write(Document doc) { write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc); } public void write(FieldBase field, Document doc) { - //save the starting position in the buffer - int startPos = buf.position(); - buf.putShort(Document.SERIALIZED_VERSION); + //save the starting position in the buffer + int lenPos = buf.position(); // Temporary length, fill in after serialization is done. buf.putInt(0); doc.getId().serialize(this); + Struct head = doc.getHeader(); + Struct body = doc.getBody(); + boolean hasHead = (head.getFieldCount() != 0); + boolean hasBody = (body.getFieldCount() != 0); + byte contents = 0x01; // Indicating we have document type which we always have - if (doc.getHeader().getFieldCount() > 0) { + if (hasHead) { contents |= 0x2; // Indicate we have header } - if (!headerOnly && doc.getBody().getFieldCount() > 0) { + if (hasBody) { contents |= 0x4; // Indicate we have a body } buf.put(contents); doc.getDataType().serialize(this); - - if (doc.getHeader().getFieldCount() > 0) { - doc.getHeader().serialize(doc.getDataType().getField("header"), this); + if (hasHead) { + head.serialize(null, this); } - - if (!headerOnly && doc.getBody().getFieldCount() > 0) { - doc.getBody().serialize(doc.getDataType().getField("body"), this); + if (hasBody) { + body.serialize(null, this); } - int finalPos = buf.position(); - - buf.position(startPos + 2); - buf.putInt(finalPos - startPos - 2 - 4); // Don't include the length itself or the version + buf.position(lenPos); + buf.putInt(finalPos - lenPos - 4); // Don't include the length itself or the version buf.position(finalPos); - } /** diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java new file mode 100644 index 00000000000..7b7878083a2 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -0,0 +1,709 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.serialization; + +import com.yahoo.compress.Compressor; + +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.annotation.AlternateSpanList; +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationReference; +import com.yahoo.document.annotation.Span; +import com.yahoo.document.annotation.SpanList; +import com.yahoo.document.annotation.SpanNode; +import com.yahoo.document.annotation.SpanTree; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.BoolFieldValue; +import com.yahoo.document.datatypes.ByteFieldValue; +import com.yahoo.document.datatypes.CollectionFieldValue; +import com.yahoo.document.datatypes.DoubleFieldValue; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.FloatFieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.document.datatypes.MapFieldValue; +import com.yahoo.document.datatypes.PredicateFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.ReferenceFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; +import com.yahoo.document.fieldpathupdate.FieldPathUpdate; +import com.yahoo.document.predicate.BinaryFormat; +import com.yahoo.document.update.AddValueUpdate; +import com.yahoo.document.update.ArithmeticValueUpdate; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.ClearValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.MapValueUpdate; +import com.yahoo.document.update.RemoveValueUpdate; +import com.yahoo.document.update.ValueUpdate; +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import com.yahoo.vespa.objects.BufferSerializer; +import com.yahoo.vespa.objects.FieldBase; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import static com.yahoo.text.Utf8.calculateBytePositions; + +/** + * Class used for serializing documents on the Vespa 6.x document format. + * + * @author baldersheim + **/ +public class VespaDocumentSerializer6 extends BufferSerializer implements DocumentSerializer { + + private int spanNodeCounter = -1; + private int[] bytePositions; + + VespaDocumentSerializer6(GrowableByteBuffer buf) { + super(buf); + } + + public void write(Document doc) { + write(new Field(doc.getDataType().getName(), 0, doc.getDataType(), true), doc); + } + + @SuppressWarnings("deprecation") + public void write(FieldBase field, Document doc) { + buf.putShort(Document.SERIALIZED_VERSION); + + //save the position of the length in the buffer + int lenPos = buf.position(); + // Temporary length, fill in after serialization is done. + buf.putInt(0); + + doc.getId().serialize(this); + + Struct head = doc.getHeader(); + Struct body = doc.getBody(); + boolean hasHead = (head.getFieldCount() != 0); + boolean hasBody = (body.getFieldCount() != 0); + + byte contents = 0x01; // Indicating we have document type which we always have + if (hasHead) { + contents |= 0x2; // Indicate we have header + } + if (hasBody) { + contents |= 0x4; // Indicate we have a body + } + buf.put(contents); + + doc.getDataType().serialize(this); + if (hasHead) { + head.serialize(null, this); + } + if (hasBody) { + body.serialize(null, this); + } + int finalPos = buf.position(); + buf.position(lenPos); + buf.putInt(finalPos - lenPos - 4); // Don't include the length itself or the version + buf.position(finalPos); + } + + /** + * Write out the value of field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, FieldValue value) { + throw new IllegalArgumentException("Not Implemented"); + } + + /** + * Write out the value of array field + * + * @param field - field description (name and data type) + * @param array - field value + */ + public <T extends FieldValue> void write(FieldBase field, Array<T> array) { + buf.putInt1_2_4Bytes(array.size()); + + List<T> lst = array.getValues(); + for (FieldValue value : lst) { + value.serialize(this); + } + + } + + public <K extends FieldValue, V extends FieldValue> void write(FieldBase field, MapFieldValue<K, V> map) { + buf.putInt1_2_4Bytes(map.size()); + for (Map.Entry<K, V> e : map.entrySet()) { + e.getKey().serialize(this); + e.getValue().serialize(this); + } + } + + /** + * Write out the value of byte field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, ByteFieldValue value) { + buf.put(value.getByte()); + } + + @Override + public void write(FieldBase field, BoolFieldValue value) { + byte asByte = value.getBoolean() ? (byte)1 : (byte)0; + buf.put(asByte); + } + + /** + * Write out the value of collection field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public <T extends FieldValue> void write(FieldBase field, CollectionFieldValue<T> value) { + throw new IllegalArgumentException("Not Implemented"); + } + + /** + * Write out the value of double field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, DoubleFieldValue value) { + buf.putDouble(value.getDouble()); + } + + /** + * Write out the value of float field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, FloatFieldValue value) { + buf.putFloat(value.getFloat()); + } + + /** + * Write out the value of integer field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, IntegerFieldValue value) { + buf.putInt(value.getInteger()); + } + + /** + * Write out the value of long field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, LongFieldValue value) { + buf.putLong(value.getLong()); + } + + /** + * Write out the value of raw field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, Raw value) { + ByteBuffer rawBuf = value.getByteBuffer(); + int origPos = rawBuf.position(); + buf.putInt(rawBuf.remaining()); + buf.put(rawBuf); + rawBuf.position(origPos); + + } + + @Override + public void write(FieldBase field, PredicateFieldValue value) { + byte[] buf = BinaryFormat.encode(value.getPredicate()); + this.buf.putInt(buf.length); + this.buf.put(buf); + } + + /** + * Write out the value of string field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, StringFieldValue value) { + byte[] stringBytes = createUTF8CharArray(value.getString()); + + byte coding = 0; + //Use bit 6 of "coding" to say whether span tree is available or not + if (!value.getSpanTrees().isEmpty()) { + coding |= 64; + } + buf.put(coding); + buf.putInt1_4Bytes(stringBytes.length + 1); + + buf.put(stringBytes); + buf.put(((byte) 0)); + + Map<String, SpanTree> trees = value.getSpanTreeMap(); + if ((trees != null) && !trees.isEmpty()) { + try { + //we don't support serialization of nested span trees, so this is safe: + bytePositions = calculateBytePositions(value.getString()); + //total length. record position and go back here if necessary: + int posBeforeSize = buf.position(); + buf.putInt(0); + buf.putInt1_2_4Bytes(trees.size()); + + for (SpanTree tree : trees.values()) { + try { + write(tree); + } catch (SerializationException e) { + throw e; + } catch (RuntimeException e) { + throw new SerializationException("Exception thrown while serializing span tree '" + + tree.getName() + "'; string='" + value.getString() + "'", e); + } + } + int endPos = buf.position(); + buf.position(posBeforeSize); + buf.putInt(endPos - posBeforeSize - 4); //length shall exclude itself + buf.position(endPos); + } finally { + bytePositions = null; + } + } + } + + @Override + public void write(FieldBase field, TensorFieldValue value) { + if (value.getTensor().isPresent()) { + byte[] encodedTensor = TypedBinaryFormat.encode(value.getTensor().get()); + buf.putInt1_4Bytes(encodedTensor.length); + buf.put(encodedTensor); + } else { + buf.putInt1_4Bytes(0); + } + } + + @Override + public void write(FieldBase field, ReferenceFieldValue value) { + if (value.getDocumentId().isPresent()) { + // We piggyback on DocumentId's existing serialization code, but need to know + // whether or not it's present or merely the empty string. + buf.put((byte)1); + write(value.getDocumentId().get()); + } else { + buf.put((byte)0); + } + } + + /** + * Write out the value of struct field + * + * @param field - field description (name and data type) + * @param s - field value + */ + public void write(FieldBase field, Struct s) { + // Serialize all parts first.. As we need to know length before starting + // Serialize all the fields. + + //keep the buffer we're serializing everything into: + GrowableByteBuffer bigBuffer = buf; + + //create a new buffer and serialize into that for a while: + GrowableByteBuffer buffer = new GrowableByteBuffer(4096, 2.0f); + buf = buffer; + + List<Integer> fieldIds = new LinkedList<>(); + List<java.lang.Integer> fieldLengths = new LinkedList<>(); + + for (Map.Entry<Field, FieldValue> value : s.getFields()) { + + int startPos = buffer.position(); + value.getValue().serialize(value.getKey(), this); + + fieldLengths.add(buffer.position() - startPos); + fieldIds.add(value.getKey().getId(s.getVersion())); + } + + // Switch buffers again: + buffer.flip(); + buf = bigBuffer; + + int uncompressedSize = buffer.remaining(); + Compressor.Compression compression = + s.getDataType().getCompressor().compress(buffer.getByteBuffer().array(), buffer.remaining()); + + // Actual serialization starts here. + int lenPos = buf.position(); + putInt(null, 0); // Move back to this after compression is done. + buf.put(compression.type().getCode()); + + if (compression.data() != null && compression.type().isCompressed()) { + buf.putInt2_4_8Bytes(uncompressedSize); + } + + buf.putInt1_4Bytes(s.getFieldCount()); + + for (int i = 0; i < s.getFieldCount(); ++i) { + putInt1_4Bytes(null, fieldIds.get(i)); + putInt2_4_8Bytes(null, fieldLengths.get(i)); + } + + int pos = buf.position(); + if (compression.data() != null && compression.type().isCompressed()) { + put(null, compression.data()); + } else { + put(null, buffer.getByteBuffer()); + } + int dataLength = buf.position() - pos; + + int posNow = buf.position(); + buf.position(lenPos); + putInt(null, dataLength); + buf.position(posNow); + } + + /** + * Write out the value of structured field + * + * @param field - field description (name and data type) + * @param value - field value + */ + public void write(FieldBase field, StructuredFieldValue value) { + throw new IllegalArgumentException("Not Implemented"); + } + + /** + * Write out the value of weighted set field + * + * @param field - field description (name and data type) + * @param ws - field value + */ + public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> ws) { + WeightedSetDataType type = ws.getDataType(); + putInt(null, type.getNestedType().getId()); + putInt(null, ws.size()); + + Iterator<T> it = ws.fieldValueIterator(); + while (it.hasNext()) { + FieldValue key = it.next(); + java.lang.Integer value = ws.get(key); + int sizePos = buf.position(); + putInt(null, 0); + int startPos = buf.position(); + key.serialize(this); + putInt(null, value); + int finalPos = buf.position(); + int size = finalPos - startPos; + buf.position(sizePos); + putInt(null, size); + buf.position(finalPos); + } + + } + + public void write(FieldBase field, AnnotationReference value) { + int annotationId = value.getReference().getScratchId(); + if (annotationId >= 0) { + buf.putInt1_2_4Bytes(annotationId); + } else { + throw new SerializationException("Could not serialize AnnotationReference value, reference not found (" + value + ")"); + } + } + + public void write(DocumentId id) { + put(null, id.getScheme().toUtf8().getBytes()); + putByte(null, (byte) 0); + } + + public void write(DocumentType type) { + byte[] docType = createUTF8CharArray(type.getName()); + put(null, docType); + putByte(null, ((byte) 0)); + putShort(null, (short) 0); // Used to hold the version. Is now always 0. + } + + public void write(Annotation annotation) { + buf.putInt(annotation.getType().getId()); //name hash + + byte features = 0; + if (annotation.isSpanNodeValid()) { + features |= ((byte) 1); + } + if (annotation.hasFieldValue()) { + features |= ((byte) 2); + } + buf.put(features); + + int posBeforeSize = buf.position(); + buf.putInt1_2_4BytesAs4(0); + + //write ID of span node: + if (annotation.isSpanNodeValid()) { + int spanNodeId = annotation.getSpanNode().getScratchId(); + if (spanNodeId >= 0) { + buf.putInt1_2_4Bytes(spanNodeId); + } else { + throw new SerializationException("Could not serialize annotation, associated SpanNode not found (" + annotation + ")"); + } + } + + //write annotation value: + if (annotation.hasFieldValue()) { + buf.putInt(annotation.getType().getDataType().getId()); + annotation.getFieldValue().serialize(this); + } + + int end = buf.position(); + buf.position(posBeforeSize); + buf.putInt1_2_4BytesAs4(end - posBeforeSize - 4); + buf.position(end); + } + + public void write(SpanTree tree) { + //we don't support serialization of nested span trees: + if (spanNodeCounter >= 0) { + throw new SerializationException("Serialization of nested SpanTrees is not supported."); + } + + //we're going to write a new SpanTree, create a new Map for nodes: + spanNodeCounter = 0; + + //make sure tree is consistent before continuing: + tree.cleanup(); + + try { + new StringFieldValue(tree.getName()).serialize(this); + + write(tree.getRoot()); + { + //add all annotations to temporary list and sort it, to get predictable serialization + List<Annotation> tmpAnnotationList = new ArrayList<Annotation>(tree.numAnnotations()); + for (Annotation annotation : tree) { + tmpAnnotationList.add(annotation); + } + Collections.sort(tmpAnnotationList); + + int annotationCounter = 0; + //add all annotations to map here, in case of back-references: + for (Annotation annotation : tmpAnnotationList) { + annotation.setScratchId(annotationCounter++); + } + + buf.putInt1_2_4Bytes(tmpAnnotationList.size()); + for (Annotation annotation : tmpAnnotationList) { + write(annotation); + } + } + } finally { + //we're done, let's set these to null to save memory and prevent madness: + spanNodeCounter = -1; + } + } + + public void write(SpanNode spanNode) { + if (spanNodeCounter >= 0) { + spanNode.setScratchId(spanNodeCounter++); + } + if (spanNode instanceof Span) { + write((Span) spanNode); + } else if (spanNode instanceof AlternateSpanList) { + write((AlternateSpanList) spanNode); + } else if (spanNode instanceof SpanList) { + write((SpanList) spanNode); + } else { + throw new IllegalStateException("BUG!! Unable to serialize " + spanNode); + } + } + + public void write(Span span) { + buf.put(Span.ID); + + if (bytePositions != null) { + int byteFrom = bytePositions[span.getFrom()]; + int byteLength = bytePositions[span.getFrom() + span.getLength()] - byteFrom; + + buf.putInt1_2_4Bytes(byteFrom); + buf.putInt1_2_4Bytes(byteLength); + } else { + throw new SerializationException("Cannot serialize Span " + span + ", no access to parent StringFieldValue."); + } + } + + public void write(SpanList spanList) { + buf.put(SpanList.ID); + buf.putInt1_2_4Bytes(spanList.numChildren()); + Iterator<SpanNode> children = spanList.childIterator(); + while (children.hasNext()) { + write(children.next()); + } + } + + public void write(AlternateSpanList altSpanList) { + buf.put(AlternateSpanList.ID); + buf.putInt1_2_4Bytes(altSpanList.getNumSubTrees()); + for (int i = 0; i < altSpanList.getNumSubTrees(); i++) { + buf.putDouble(altSpanList.getProbability(i)); + buf.putInt1_2_4Bytes(altSpanList.numChildren(i)); + Iterator<SpanNode> children = altSpanList.childIterator(i); + while (children.hasNext()) { + write(children.next()); + } + } + } + + @Override + public void write(DocumentUpdate update) { + update.getId().serialize(this); + + update.getDocumentType().serialize(this); + + putInt(null, update.fieldUpdates().size()); + + for (FieldUpdate up : update.fieldUpdates()) { + up.serialize(this); + } + + DocumentUpdateFlags flags = new DocumentUpdateFlags(); + flags.setCreateIfNonExistent(update.getCreateIfNonExistent()); + putInt(null, flags.injectInto(update.fieldPathUpdates().size())); + + for (FieldPathUpdate up : update.fieldPathUpdates()) { + up.serialize(this); + } + } + + public void write(FieldPathUpdate update) { + putByte(null, (byte)update.getUpdateType().getCode()); + put(null, update.getOriginalFieldPath()); + put(null, update.getOriginalWhereClause()); + } + + public void write(AssignFieldPathUpdate update) { + write((FieldPathUpdate)update); + byte flags = 0; + if (update.getRemoveIfZero()) { + flags |= AssignFieldPathUpdate.REMOVE_IF_ZERO; + } + if (update.getCreateMissingPath()) { + flags |= AssignFieldPathUpdate.CREATE_MISSING_PATH; + } + if (update.isArithmetic()) { + flags |= AssignFieldPathUpdate.ARITHMETIC_EXPRESSION; + putByte(null, flags); + put(null, update.getExpression()); + } else { + putByte(null, flags); + update.getFieldValue().serialize(this); + } + } + + public void write(AddFieldPathUpdate update) { + write((FieldPathUpdate)update); + update.getNewValues().serialize(this); + } + + @Override + public void write(FieldUpdate update) { + putInt(null, update.getField().getId(Document.SERIALIZED_VERSION)); + putInt(null, update.getValueUpdates().size()); + for (ValueUpdate vupd : update.getValueUpdates()) { + putInt(null, vupd.getValueUpdateClassID().id); + vupd.serialize(this, update.getField().getDataType()); + } + } + + @Override + public void write(AddValueUpdate update, DataType superType) { + writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); + putInt(null, update.getWeight()); + } + + @Override + public void write(MapValueUpdate update, DataType superType) { + if (superType instanceof ArrayDataType) { + CollectionDataType type = (CollectionDataType) superType; + IntegerFieldValue index = (IntegerFieldValue) update.getValue(); + index.serialize(this); + putInt(null, update.getUpdate().getValueUpdateClassID().id); + update.getUpdate().serialize(this, type.getNestedType()); + } else if (superType instanceof WeightedSetDataType) { + writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); + putInt(null, update.getUpdate().getValueUpdateClassID().id); + update.getUpdate().serialize(this, DataType.INT); + } else { + throw new SerializationException("MapValueUpdate only works for arrays and weighted sets"); + } + } + + @Override + public void write(ArithmeticValueUpdate update) { + putInt(null, update.getOperator().id); + putDouble(null, update.getOperand().doubleValue()); + } + + @Override + public void write(AssignValueUpdate update, DataType superType) { + if (update.getValue() == null) { + putByte(null, (byte) 0); + } else { + putByte(null, (byte) 1); + writeValue(this, superType, update.getValue()); + } + } + + @Override + public void write(RemoveValueUpdate update, DataType superType) { + writeValue(this, ((CollectionDataType)superType).getNestedType(), update.getValue()); + } + + @Override + public void write(ClearValueUpdate clearValueUpdate, DataType superType) { + //TODO: This has never ever been implemented. Has this ever worked? + } + + /** + * Returns the serialized size of the given {@link Document}. Please note that this method performs actual + * serialization of the document, but simply return the size of the final {@link GrowableByteBuffer}. If you need + * the buffer itself, do NOT use this method. + * + * @param doc The Document whose size to calculate. + * @return The size in bytes. + */ + public static long getSerializedSize(Document doc) { + DocumentSerializer serializer = new VespaDocumentSerializer6(new GrowableByteBuffer()); + serializer.write(doc); + return serializer.getBuf().position(); + } + + private static void writeValue(VespaDocumentSerializer6 serializer, DataType dataType, Object value) { + FieldValue fieldValue; + if (value instanceof FieldValue) { + fieldValue = (FieldValue)value; + } else { + fieldValue = dataType.createFieldValue(value); + } + fieldValue.serialize(serializer); + } + +} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java index 92bce41ba8c..bfa746a02b1 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializerHead.java @@ -1,80 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.serialization; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.document.datatypes.ByteFieldValue; -import com.yahoo.document.fieldpathupdate.AddFieldPathUpdate; -import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; -import com.yahoo.document.fieldpathupdate.FieldPathUpdate; -import com.yahoo.document.update.FieldUpdate; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.vespa.objects.FieldBase; /** * Class used for serializing documents on the current head document format. * * @author baldersheim */ -@SuppressWarnings("deprecation") -public class VespaDocumentSerializerHead extends VespaDocumentSerializer42 { +public class VespaDocumentSerializerHead extends VespaDocumentSerializer6 { public VespaDocumentSerializerHead(GrowableByteBuffer buf) { super(buf); } - @Override - public void write(DocumentUpdate update) { - update.getId().serialize(this); - - update.getDocumentType().serialize(this); - - putInt(null, update.fieldUpdates().size()); - - for (FieldUpdate up : update.fieldUpdates()) { - up.serialize(this); - } - - DocumentUpdateFlags flags = new DocumentUpdateFlags(); - flags.setCreateIfNonExistent(update.getCreateIfNonExistent()); - putInt(null, flags.injectInto(update.fieldPathUpdates().size())); - - for (FieldPathUpdate up : update.fieldPathUpdates()) { - up.serialize(this); - } - } - - public void write(FieldPathUpdate update) { - putByte(null, (byte)update.getUpdateType().getCode()); - put(null, update.getOriginalFieldPath()); - put(null, update.getOriginalWhereClause()); - } - - public void write(AssignFieldPathUpdate update) { - write((FieldPathUpdate)update); - byte flags = 0; - if (update.getRemoveIfZero()) { - flags |= AssignFieldPathUpdate.REMOVE_IF_ZERO; - } - if (update.getCreateMissingPath()) { - flags |= AssignFieldPathUpdate.CREATE_MISSING_PATH; - } - if (update.isArithmetic()) { - flags |= AssignFieldPathUpdate.ARITHMETIC_EXPRESSION; - putByte(null, flags); - put(null, update.getExpression()); - } else { - putByte(null, flags); - update.getFieldValue().serialize(this); - } - } - - public void write(AddFieldPathUpdate update) { - write((FieldPathUpdate)update); - update.getNewValues().serialize(this); - } - - @Override - public void write(FieldBase field, ByteFieldValue value) { - buf.put(value.getByte()); - } } diff --git a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java index 163bda5b623..192afba80b3 100644 --- a/document/src/main/java/com/yahoo/document/update/FieldUpdate.java +++ b/document/src/main/java/com/yahoo/document/update/FieldUpdate.java @@ -291,7 +291,7 @@ public class FieldUpdate { } public final void serialize(GrowableByteBuffer buf) { - serialize(DocumentSerializerFactory.create42(buf)); + serialize(DocumentSerializerFactory.create6(buf)); } public void serialize(DocumentUpdateWriter data) { diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java index ba33b6a83ad..68116e777d6 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// TODO: Remove this package on Vespa 7 +// TODO: Remove this package on Vespa 8 @ExportPackage package com.yahoo.vespaxmlparser; diff --git a/document/src/test/java/com/yahoo/document/DocInDocTestCase.java b/document/src/test/java/com/yahoo/document/DocInDocTestCase.java index cdc868af987..f5c158595d2 100644 --- a/document/src/test/java/com/yahoo/document/DocInDocTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocInDocTestCase.java @@ -37,13 +37,13 @@ public class DocInDocTestCase { Document outer = new Document(manager.getDocumentType("outerdoc"), "doc:outer:the:only:one"); outer.setFieldValue("innerdocuments", innerArray); - DocumentSerializer serializer = DocumentSerializerFactory.create42(); + DocumentSerializer serializer = DocumentSerializerFactory.create6(); serializer.write(outer); GrowableByteBuffer buf = serializer.getBuf(); buf.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buf); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(manager, buf); Document outerDeserialized = new Document(deserializer); assertEquals(outer, outerDeserialized); diff --git a/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java index ce55a5a501c..d5fc1386900 100755 --- a/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentPathUpdateTestCase.java @@ -581,9 +581,9 @@ public class DocumentPathUpdateTestCase { docUp.addFieldPathUpdate(ass); GrowableByteBuffer buffer = new GrowableByteBuffer(); - docUp.serialize(DocumentSerializerFactory.createHead(buffer)); + docUp.serialize(DocumentSerializerFactory.create6(buffer)); buffer.flip(); - DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer)); + DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.create6(docMan, buffer)); assertEquals(docUp, docUp2); } @@ -599,9 +599,9 @@ public class DocumentPathUpdateTestCase { docUp.addFieldPathUpdate(add); GrowableByteBuffer buffer = new GrowableByteBuffer(); - docUp.serialize(DocumentSerializerFactory.createHead(buffer)); + docUp.serialize(DocumentSerializerFactory.create6(buffer)); buffer.flip(); - DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer)); + DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.create6(docMan, buffer)); assertEquals(docUp, docUp2); } @@ -613,9 +613,9 @@ public class DocumentPathUpdateTestCase { docUp.addFieldPathUpdate(remove); GrowableByteBuffer buffer = new GrowableByteBuffer(); - docUp.serialize(DocumentSerializerFactory.createHead(buffer)); + docUp.serialize(DocumentSerializerFactory.create6(buffer)); buffer.flip(); - DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.createHead(docMan, buffer)); + DocumentUpdate docUp2 = new DocumentUpdate(DocumentDeserializerFactory.create6(docMan, buffer)); assertEquals(docUp, docUp2); } @@ -658,7 +658,7 @@ public class DocumentPathUpdateTestCase { DocumentUpdate docUp = createDocumentUpdateForSerialization(); GrowableByteBuffer buffer = new GrowableByteBuffer(); - docUp.serialize(DocumentSerializerFactory.createHead(buffer)); + docUp.serialize(DocumentSerializerFactory.create6(buffer)); int size = buffer.position(); buffer.position(0); @@ -672,7 +672,7 @@ public class DocumentPathUpdateTestCase { public void testReadSerializedFile() throws IOException { docMan = DocumentTestCase.setUpCppDocType(); byte[] data = DocumentTestCase.readFile("src/tests/data/serialize-fieldpathupdate-cpp.dat"); - DocumentDeserializer buf = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data)); + DocumentDeserializer buf = DocumentDeserializerFactory.create6(docMan, GrowableByteBuffer.wrap(data)); DocumentUpdate upd = new DocumentUpdate(buf); diff --git a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java index 7d1992225e4..8ff168adc4b 100644 --- a/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentSerializationTestCase.java @@ -170,7 +170,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { } System.err.println("Checking doc from file " + test.testFile); - Document doc = new Document(DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(buffer))); + Document doc = new Document(DocumentDeserializerFactory.create6(docMan, GrowableByteBuffer.wrap(buffer))); System.err.println("Id: " + doc.getId()); @@ -214,7 +214,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { doc.setFieldValue("friend", 2384L); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(doc); buffer.flip(); @@ -222,7 +222,7 @@ public class DocumentSerializationTestCase extends AbstractTypesTest { fos.write(buffer.array(), 0, buffer.limit()); fos.close(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); Document doc2 = new Document(deserializer); assertEquals(doc, doc2); diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 3eebc4396e8..28267a22a56 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -708,54 +708,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - public void testCppDocSplit() throws IOException { - docMan = setUpCppDocType(); - byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat"); - byte[] bodyData = readFile("src/test/document/serializecppsplit_body.dat"); - - DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData), - GrowableByteBuffer.wrap(bodyData)); - - Document doc = new Document(header); - - assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString()); - assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield")); - assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield")); - assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield")); - assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield")); - assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield")); - assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield")); - //NOTE: The value really is unsigned 254, which becomes signed -2: - assertEquals(new ByteFieldValue((byte)-2), doc.getFieldValue("bytefield")); - ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes()); - assertEquals(new Raw(raw), doc.getFieldValue("rawfield")); - - Document docindoc = (Document)doc.getFieldValue("docfield"); - assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType()); - assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId()); - - WeightedSet wset = (WeightedSet)doc.getFieldValue("wsfield"); - assertEquals(Integer.valueOf(50), wset.get(new StringFieldValue("Weighted 0"))); - assertEquals(Integer.valueOf(199), wset.get(new StringFieldValue("Weighted 1"))); - } - - @Test - public void testCppDocSplitNoBody() throws IOException { - docMan = setUpCppDocType(); - byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat"); - - DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData)); - - Document doc = new Document(header); - - assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString()); - assertEquals(new FloatFieldValue((float)-9.23), doc.getFieldValue("floatfield")); - assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield")); - assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield")); - assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield")); - } - - @Test @SuppressWarnings("deprecation") public void testGenerateSerializedFile() throws IOException { @@ -862,48 +814,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); - // Do the same thing, splitting document in two - DocumentSerializer header = DocumentSerializerFactory.create42(new GrowableByteBuffer(), true); - DocumentSerializer body = DocumentSerializerFactory.create42(new GrowableByteBuffer()); - doc.serializeHeader(header); - doc.serializeBody(body); - header.getBuf().flip(); - body.getBuf().flip(); - - try { - FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat"); - fos.write(header.getBuf().array(), 0, header.getBuf().remaining()); - fos.close(); - fos = new FileOutputStream("src/test/files/testser-split.body.dat"); - fos.write(body.getBuf().array(), 0, body.getBuf().remaining()); - fos.close(); - } catch (Exception e) { - } - - DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf()); - - doc2 = new Document(deser); - - assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid")); - assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date")); - assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from")); - assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to")); - assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject")); - assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body")); - assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount")); - assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments")); - docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array(); - doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array(); - assertEquals(docRawBytes.length, doc2RawBytes.length); - for (int i = 0; i < docRawBytes.length; i++) { - assertEquals(docRawBytes[i], doc2RawBytes[i]); - } - assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); - assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); - - Document docInDoc = (Document)doc.getFieldValue("docindoc"); - assert (docInDoc != null); - assertEquals(new StringFieldValue("ball"), docInDoc.getFieldValue("tull")); } @Test @@ -953,45 +863,6 @@ public class DocumentTestCase extends DocumentTestCaseBase { } assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); - - // Do the same thing, splitting document in two - BufferSerializer header = new BufferSerializer(new GrowableByteBuffer()); - BufferSerializer body = new BufferSerializer(new GrowableByteBuffer()); - doc.serializeHeader(header); - doc.serializeBody(body); - header.getBuf().flip(); - body.getBuf().flip(); - - try { - FileOutputStream fos = new FileOutputStream("src/test/files/testser-split.header.dat"); - fos.write(header.getBuf().array(), 0, header.getBuf().remaining()); - fos.close(); - fos = new FileOutputStream("src/test/files/testser-split.body.dat"); - fos.write(body.getBuf().array(), 0, body.getBuf().remaining()); - fos.close(); - } catch (Exception e) { - } - - DocumentDeserializer deser = DocumentDeserializerFactory.create42(docMan, header.getBuf(), body.getBuf()); - - doc2 = new Document(deser); - - assertEquals(doc.getFieldValue("mailid"), doc2.getFieldValue("mailid")); - assertEquals(doc.getFieldValue("date"), doc2.getFieldValue("date")); - assertEquals(doc.getFieldValue("from"), doc2.getFieldValue("from")); - assertEquals(doc.getFieldValue("to"), doc2.getFieldValue("to")); - assertEquals(doc.getFieldValue("subject"), doc2.getFieldValue("subject")); - assertEquals(doc.getFieldValue("body"), doc2.getFieldValue("body")); - assertEquals(doc.getFieldValue("attachmentcount"), doc2.getFieldValue("attachmentcount")); - assertEquals(doc.getFieldValue("attachments"), doc2.getFieldValue("attachments")); - docRawBytes = ((Raw)doc.getFieldValue("rawfield")).getByteBuffer().array(); - doc2RawBytes = ((Raw)doc2.getFieldValue("rawfield")).getByteBuffer().array(); - assertEquals(docRawBytes.length, doc2RawBytes.length); - for (int i = 0; i < docRawBytes.length; i++) { - assertEquals(docRawBytes[i], doc2RawBytes[i]); - } - assertEquals(doc.getFieldValue("weightedfield"), doc2.getFieldValue("weightedfield")); - assertEquals(doc.getFieldValue("mapfield"), doc2.getFieldValue("mapfield")); } @Test @@ -1000,7 +871,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { BufferSerializer buf = new BufferSerializer(); try { - new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf())); + new Document(DocumentDeserializerFactory.create6(docMan, buf.getBuf())); assertTrue(false); } catch (Exception e) { assertTrue(true); @@ -1008,7 +879,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { buf = BufferSerializer.wrap("Hello world".getBytes()); try { - new Document(DocumentDeserializerFactory.create42(docMan, buf.getBuf())); + new Document(DocumentDeserializerFactory.create6(docMan, buf.getBuf())); assertTrue(false); } catch (Exception e) { assertTrue(true); @@ -1450,11 +1321,11 @@ public class DocumentTestCase extends DocumentTestCaseBase { } public void serialize(String docId) { new Document(docType, DocumentId.createFromSerialized(docId)) - .serialize(DocumentSerializerFactory.createHead(buffer)); + .serialize(DocumentSerializerFactory.create6(buffer)); buffer.flip(); } public Document deserialize() { - return new Document(DocumentDeserializerFactory.createHead(docMan, buffer)); + return new Document(DocumentDeserializerFactory.create6(docMan, buffer)); } } diff --git a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java index a9f77cb5eb0..b86094e5a4d 100644 --- a/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentUpdateTestCase.java @@ -299,13 +299,13 @@ public class DocumentUpdateTestCase { docUp.addFieldUpdate(assignSingle); docUp.addFieldUpdate(addMultiList); - DocumentDeserializer buf = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer()); - docUp.serialize((DocumentUpdateWriter)buf); - buf.getBuf().flip(); + GrowableByteBuffer buf = new GrowableByteBuffer(); + docUp.serialize(DocumentSerializerFactory.create42(buf)); + buf.flip(); try { FileOutputStream fos = new FileOutputStream("src/test/files/updateser.dat"); - fos.write(buf.getBuf().array(), 0, buf.getBuf().remaining()); + fos.write(buf.array(), 0, buf.remaining()); fos.close(); } catch (Exception e) { } @@ -330,10 +330,9 @@ public class DocumentUpdateTestCase { + (4 //valueUpdateClassID + (4 + 4 + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1) + 4 + (1 + 1 + 2 + 1))))) //value - , buf.getBuf().remaining()); - - DocumentUpdate docUpDeser = new DocumentUpdate(buf); + , buf.remaining()); + DocumentUpdate docUpDeser = new DocumentUpdate(DocumentDeserializerFactory.create42(docMan, buf)); assertEquals(docUp.getDocumentType(), docUpDeser.getDocumentType()); assertEquals(docUp, docUpDeser); } @@ -402,7 +401,7 @@ public class DocumentUpdateTestCase { upd.addFieldUpdate(serAdd); GrowableByteBuffer buf = new GrowableByteBuffer(100, 2.0f); - upd.serialize(buf); + upd.serialize(DocumentSerializerFactory.create42(buf)); int size = buf.position(); buf.position(0); @@ -630,11 +629,11 @@ public class DocumentUpdateTestCase { public void testRequireThatCreateIfNonExistentFlagIsSerializedAndDeserialized() { docUp.setCreateIfNonExistent(true); - DocumentSerializer serializer = DocumentSerializerFactory.createHead(new GrowableByteBuffer()); + DocumentSerializer serializer = DocumentSerializerFactory.create6(new GrowableByteBuffer()); docUp.serialize(serializer); serializer.getBuf().flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.createHead(docMan, serializer.getBuf()); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, serializer.getBuf()); DocumentUpdate deserialized = new DocumentUpdate(deserializer); assertEquals(docUp, deserialized); assertTrue(deserialized.getCreateIfNonExistent()); @@ -655,11 +654,11 @@ public class DocumentUpdateTestCase { @Test public void testThatAssignValueUpdateForTensorFieldCanBeSerializedAndDeserialized() { DocumentUpdate serializedUpdate = createTensorAssignUpdate(); - DocumentSerializer serializer = DocumentSerializerFactory.createHead(new GrowableByteBuffer()); + DocumentSerializer serializer = DocumentSerializerFactory.create6(new GrowableByteBuffer()); serializedUpdate.serialize(serializer); serializer.getBuf().flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.createHead(docMan, serializer.getBuf()); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, serializer.getBuf()); DocumentUpdate deserializedUpdate = new DocumentUpdate(deserializer); assertEquals(serializedUpdate, deserializedUpdate); } diff --git a/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java index 47e514d74f5..186766f799b 100755 --- a/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/AlternateSpanListTestCase.java @@ -57,7 +57,7 @@ public class AlternateSpanListTestCase extends AbstractTypesTest { GrowableByteBuffer buffer; { buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk"); SpanTree tree = new SpanTree("bababa", alternateSpanList); value.setSpanTree(tree); @@ -66,7 +66,7 @@ public class AlternateSpanListTestCase extends AbstractTypesTest { } AlternateSpanList alternateSpanList2; { - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue value = new StringFieldValue(); deserializer.read(null, value); alternateSpanList2 = (AlternateSpanList)value.getSpanTree("bababa").getRoot(); diff --git a/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java b/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java index de674aa3cca..cb04ca1de62 100644 --- a/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/AnnotationTestCase.java @@ -115,11 +115,11 @@ public class AnnotationTestCase extends AbstractTypesTest { private void serializeAndAssert(Annotation annotation) { GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(annotation); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); Annotation annotation2 = new Annotation(); deserializer.read(annotation2); diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java index 18f19030c9e..8b9649d697f 100644 --- a/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/Bug6394548TestCase.java @@ -43,11 +43,11 @@ public class Bug6394548TestCase { String annotationsBefore = dumpAllAnnotations(tree); GrowableByteBuffer buffer = new GrowableByteBuffer(); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(doc); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(manager, buffer); Document doc2 = new Document(deserializer); System.out.println(doc2.toXml()); diff --git a/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java index 8f071650382..82b730ae4ca 100644 --- a/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/Bug6425939TestCase.java @@ -37,13 +37,13 @@ public class Bug6425939TestCase { emptyString.setSpanTree(createSpanTree()); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); Field strField = new Field("flarn", DataType.STRING); serializer.write(strField, emptyString); buffer.flip(); // Should not throw exception if bug 6425939 is fixed: - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue deserializedString = new StringFieldValue(); deserializer.read(strField, deserializedString); diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java index 887bfec5059..6f338fc47a3 100755 --- a/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/SpanListTestCase.java @@ -46,7 +46,7 @@ public class SpanListTestCase extends AbstractTypesTest { GrowableByteBuffer buffer; { buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk"); SpanTree tree = new SpanTree("bababa", spanList); value.setSpanTree(tree); @@ -55,7 +55,7 @@ public class SpanListTestCase extends AbstractTypesTest { } SpanList spanList2; { - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue value = new StringFieldValue(); deserializer.read(null, value); spanList2 = (SpanList)value.getSpanTree("bababa").getRoot(); diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java index 701f6917ecd..23d62cae239 100755 --- a/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/SpanTestCase.java @@ -79,7 +79,7 @@ public class SpanTestCase extends AbstractTypesTest { GrowableByteBuffer buffer; { buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); StringFieldValue value = new StringFieldValue("lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lkj lk"); SpanTree tree = new SpanTree("bababa", span); value.setSpanTree(tree); @@ -88,7 +88,7 @@ public class SpanTestCase extends AbstractTypesTest { } Span span2; { - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue value = new StringFieldValue(); deserializer.read(null, value); span2 = (Span)value.getSpanTree("bababa").getRoot(); diff --git a/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java index 393fe8b6688..b200e72514d 100755 --- a/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/SpanTreeTestCase.java @@ -865,12 +865,12 @@ public class SpanTreeTestCase extends AbstractTypesTest { GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(new Field("stringfield", DataType.STRING), strfval); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, buffer); StringFieldValue stringFieldValue2 = new StringFieldValue(); deserializer.read(new Field("stringfield", DataType.STRING), stringFieldValue2); diff --git a/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java b/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java index ffa71ea7939..9163b773bcf 100755 --- a/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java +++ b/document/src/test/java/com/yahoo/document/annotation/SystemTestCase.java @@ -124,10 +124,10 @@ public class SystemTestCase { annotate(inDocument); GrowableByteBuffer buffer = new GrowableByteBuffer(); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(inDocument); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(manager, buffer); Document outDocument = new Document(deserializer); consume(outDocument); diff --git a/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java index c4441910f2a..eb47fb1425d 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/MapTestCase.java @@ -144,10 +144,10 @@ public class MapTestCase { DocumentTypeManager man = new DocumentTypeManager(); man.register(mapType); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(f, map); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); MapFieldValue<FieldValue, FieldValue> map2 = new MapFieldValue<FieldValue, FieldValue>(mapType); deserializer.read(f, map2); assertNotSame(map, map2); diff --git a/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java index e9fce35ed01..296ab1ac3fc 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java +++ b/document/src/test/java/com/yahoo/document/datatypes/StringTestCase.java @@ -50,7 +50,7 @@ public class StringTestCase extends AbstractTypesTest { data.position(0); StringFieldValue tmp = new StringFieldValue(); - DocumentDeserializer deser = DocumentDeserializerFactory.create42(null, data); + DocumentDeserializer deser = DocumentDeserializerFactory.create6(null, data); tmp.deserialize(deser); java.lang.String foo2 = tmp.getString(); @@ -111,7 +111,7 @@ public class StringTestCase extends AbstractTypesTest { tmp = new StringFieldValue(); - deser = DocumentDeserializerFactory.create42(null, data); + deser = DocumentDeserializerFactory.create6(null, data); tmp.deserialize(deser); java.lang.String blah2 = tmp.getString(); @@ -129,7 +129,7 @@ public class StringTestCase extends AbstractTypesTest { data.getBuf().position(0); StringFieldValue tmp = new StringFieldValue(); - DocumentDeserializer deser = DocumentDeserializerFactory.create42(null, data.getBuf()); + DocumentDeserializer deser = DocumentDeserializerFactory.create6(null, data.getBuf()); tmp.deserialize(deser); java.lang.String test2 = tmp.getString(); assertEquals(test, test2); @@ -146,11 +146,11 @@ public class StringTestCase extends AbstractTypesTest { Field f = new Field("text", DataType.STRING); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(f, stringFieldValue); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(man, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(man, buffer); StringFieldValue stringFieldValue2 = new StringFieldValue(); deserializer.read(f, stringFieldValue2); @@ -183,7 +183,7 @@ public class StringTestCase extends AbstractTypesTest { innerTree.annotate(innerSpan, new Annotation(type)); GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); try { serializer.write(null, outerString); @@ -216,11 +216,11 @@ public class StringTestCase extends AbstractTypesTest { private Document serializeAndDeserialize(Document doc, DocumentTypeManager manager) { GrowableByteBuffer buffer = new GrowableByteBuffer(1024); - DocumentSerializer serializer = DocumentSerializerFactory.create42(buffer); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buffer); serializer.write(doc); buffer.flip(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(manager, buffer); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(manager, buffer); return new Document(deserializer); } diff --git a/document/src/test/java/com/yahoo/document/datatypes/blog.sd b/document/src/test/java/com/yahoo/document/datatypes/blog.sd index 545c46cb149..1723e942370 100644 --- a/document/src/test/java/com/yahoo/document/datatypes/blog.sd +++ b/document/src/test/java/com/yahoo/document/datatypes/blog.sd @@ -4,26 +4,19 @@ search blog { document blog { field title type string { - header indexing: summary | index - # index-to: default } field author type string { - header indexing: summary | index - # index-to: default } field body type string { - header indexing: summary | index } field url type uri { - header indexing: index | summary - # index-to: default } } diff --git a/document/src/test/java/com/yahoo/document/docindoc.sd b/document/src/test/java/com/yahoo/document/docindoc.sd index 8b6acd71f47..b05f8d71c89 100644 --- a/document/src/test/java/com/yahoo/document/docindoc.sd +++ b/document/src/test/java/com/yahoo/document/docindoc.sd @@ -1,7 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search docindoc { document docindoc { - field name type string { header } - field content type string { body } + field name type string { } + field content type string { } } } diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index b3d502ea56f..d82935e3c3b 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -323,7 +323,7 @@ public class JsonReaderTestCase { assertEquals(2, s.getFieldCount()); assertEquals(new StringFieldValue("person"), s.getFieldValue(s.getField("sandra"))); GrowableByteBuffer buf = new GrowableByteBuffer(); - DocumentSerializer serializer = DocumentSerializerFactory.createHead(buf); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buf); put.serialize(serializer); assertEquals(107, buf.position()); } @@ -345,7 +345,7 @@ public class JsonReaderTestCase { Struct s = (Struct) avu.getValue(); assertEquals(0, s.getFieldCount()); GrowableByteBuffer buf = new GrowableByteBuffer(); - DocumentSerializer serializer = DocumentSerializerFactory.createHead(buf); + DocumentSerializer serializer = DocumentSerializerFactory.create6(buf); put.serialize(serializer); assertEquals(69, buf.position()); } diff --git a/document/src/test/java/com/yahoo/document/outerdoc.sd b/document/src/test/java/com/yahoo/document/outerdoc.sd index 6fdbd402ffe..57a1bda323a 100644 --- a/document/src/test/java/com/yahoo/document/outerdoc.sd +++ b/document/src/test/java/com/yahoo/document/outerdoc.sd @@ -1,6 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. search outerdoc { document outerdoc { - field innerdocuments type array<docindoc> { body } + field innerdocuments type array<docindoc> { } } } diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java index 38cf2b0e9d1..25021b0d2f8 100644 --- a/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java +++ b/document/src/test/java/com/yahoo/document/serialization/SerializationTestUtils.java @@ -24,7 +24,7 @@ public class SerializationTestUtils { public static byte[] serializeDocument(Document doc) { GrowableByteBuffer out = new GrowableByteBuffer(); - DocumentSerializerFactory.create42(out).write(doc); + DocumentSerializerFactory.create6(out).write(doc); out.flip(); byte[] buf = new byte[out.remaining()]; out.get(buf); @@ -33,7 +33,7 @@ public class SerializationTestUtils { public static Document deserializeDocument(byte[] buf, TestDocumentFactory factory) { Document document = factory.createDocument(); - DocumentDeserializerFactory.create42(factory.typeManager(), new GrowableByteBuffer(ByteBuffer.wrap(buf))).read(document); + DocumentDeserializerFactory.create6(factory.typeManager(), new GrowableByteBuffer(ByteBuffer.wrap(buf))).read(document); return document; } diff --git a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java index 30db0af6607..653f121b2d4 100644 --- a/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/SerializeAnnotationsTestCase.java @@ -79,7 +79,7 @@ public class SerializeAnnotationsTestCase { assertEquals(serialized.limit(), serializedFromFile.limit()); StringFieldValue valueFromFile = new StringFieldValue(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile)); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, new GrowableByteBuffer(serializedFromFile)); deserializer.read(null, valueFromFile); assertEquals(value, valueFromFile); } @@ -182,7 +182,7 @@ public class SerializeAnnotationsTestCase { assertEquals(serialized.limit(), serializedFromFile.limit()); StringFieldValue valueFromFile = new StringFieldValue(); - DocumentDeserializer deserializer = DocumentDeserializerFactory.create42(docMan, new GrowableByteBuffer(serializedFromFile)); + DocumentDeserializer deserializer = DocumentDeserializerFactory.create6(docMan, new GrowableByteBuffer(serializedFromFile)); deserializer.read(null, valueFromFile); assertEquals(value, valueFromFile); } diff --git a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java index 4f34431c729..5a8d458c74b 100644 --- a/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java +++ b/document/src/test/java/com/yahoo/document/serialization/VespaDocumentSerializerTestCase.java @@ -53,7 +53,7 @@ public class VespaDocumentSerializerTestCase { PredicateFieldValue predicate = Mockito.mock(PredicateFieldValue.class); doc.setFieldValue("my_predicate", predicate); - DocumentSerializerFactory.create42(new GrowableByteBuffer()).write(doc); + DocumentSerializerFactory.create6(new GrowableByteBuffer()).write(doc); Mockito.verify(predicate, Mockito.times(1)).serialize(Mockito.same(field), Mockito.any(FieldWriter.class)); } diff --git a/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java b/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java index 3cf0fab80ae..11e0cc783af 100644 --- a/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java +++ b/document/src/test/java/com/yahoo/document/update/FieldUpdateTestCase.java @@ -186,10 +186,10 @@ public class FieldUpdateTestCase { // Copy all field updates using serialization to verify that it is supported private FieldUpdate serializedCopy(FieldUpdate source, DocumentType docType) { - DocumentSerializer buffer = DocumentSerializerFactory.create42(); + DocumentSerializer buffer = DocumentSerializerFactory.create6(); source.serialize(buffer); buffer.getBuf().flip(); - FieldUpdate copy = new FieldUpdate(DocumentDeserializerFactory.create42(docman, buffer.getBuf()), docType, Document.SERIALIZED_VERSION); + FieldUpdate copy = new FieldUpdate(DocumentDeserializerFactory.create6(docman, buffer.getBuf()), docType, Document.SERIALIZED_VERSION); assertEquals(source, copy); return copy; } diff --git a/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java b/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java index f1cce1e4c4a..05492d41b09 100644 --- a/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java +++ b/document/src/test/java/com/yahoo/document/update/SerializationTestCase.java @@ -30,7 +30,7 @@ public class SerializationTestCase { @Test public void testAddSerialization() { FieldUpdate update = FieldUpdate.createAdd(field, new StringFieldValue("value1")); - DocumentSerializer buffer = DocumentSerializerFactory.create42(); + DocumentSerializer buffer = DocumentSerializerFactory.create6(); update.serialize(buffer); buffer.getBuf().rewind(); @@ -41,18 +41,18 @@ public class SerializationTestCase { fos.close(); } catch (Exception e) {} - FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create42(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION); + FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create6(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION); assertEquals("'field1' [add value1 1]", deserializedUpdate.toString()); } @Test public void testClearSerialization() { FieldUpdate update = FieldUpdate.createClear(field); - DocumentSerializer buffer = DocumentSerializerFactory.create42(); + DocumentSerializer buffer = DocumentSerializerFactory.create6(); update.serialize(buffer); buffer.getBuf().rewind(); - FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create42(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION); + FieldUpdate deserializedUpdate = new FieldUpdate(DocumentDeserializerFactory.create6(new DocumentTypeManager(), buffer.getBuf()), documentType, Document.SERIALIZED_VERSION); assertEquals("'field1' [clear]", deserializedUpdate.toString()); } diff --git a/document/src/tests/data/serializeupdatejava.dat b/document/src/tests/data/serializeupdatejava.dat Binary files differindex e2a98d42fb1..20c56228bde 100644 --- a/document/src/tests/data/serializeupdatejava.dat +++ b/document/src/tests/data/serializeupdatejava.dat diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index b4d98f0dd21..4962b708ef5 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -553,7 +553,7 @@ void DocumentUpdateTest::testReadSerializedFile() CPPUNIT_ASSERT_EQUAL(*type, upd.getType()); // Verify assign value update. - FieldUpdate serField = upd.getUpdates()[0]; + FieldUpdate serField = upd.getUpdates()[1]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("intfield").getId()); const ValueUpdate* serValue = &serField[0]; @@ -563,7 +563,7 @@ void DocumentUpdateTest::testReadSerializedFile() CPPUNIT_ASSERT_EQUAL(IntFieldValue(4), static_cast<const IntFieldValue&>(assign->getValue())); // Verify clear field update. - serField = upd.getUpdates()[1]; + serField = upd.getUpdates()[2]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("floatfield").getId()); serValue = &serField[0]; @@ -571,7 +571,7 @@ void DocumentUpdateTest::testReadSerializedFile() CPPUNIT_ASSERT(serValue->inherits(ClearValueUpdate::classId)); // Verify add value update. - serField = upd.getUpdates()[2]; + serField = upd.getUpdates()[0]; CPPUNIT_ASSERT_EQUAL(serField.getField().getId(), type->getField("arrayoffloatfield").getId()); serValue = &serField[0]; |