diff options
Diffstat (limited to 'document/src/main/java/com')
7 files changed, 48 insertions, 13 deletions
diff --git a/document/src/main/java/com/yahoo/document/DocumentUpdate.java b/document/src/main/java/com/yahoo/document/DocumentUpdate.java index d3063b76feb..20d9b352d2d 100644 --- a/document/src/main/java/com/yahoo/document/DocumentUpdate.java +++ b/document/src/main/java/com/yahoo/document/DocumentUpdate.java @@ -147,7 +147,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP Map.Entry<Integer, FieldUpdate> entry = iter.next(); FieldUpdate update = entry.getValue(); if (!update.isEmpty()) { - ValueUpdate last = update.getValueUpdate(update.size() - 1); + ValueUpdate<?> last = update.getValueUpdate(update.size() - 1); if (last instanceof AssignValueUpdate) { FieldValue currentValue = doc.getFieldValue(update.getField()); if ((currentValue != null) && currentValue.equals(last.getValue())) { @@ -190,7 +190,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP /** Returns the type of the document this updates * - * @return The documentype of the document + * @return The document type of the document */ public DocumentType getDocumentType() { return documentType; @@ -357,9 +357,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof DocumentUpdate)) return false; - - DocumentUpdate that = (DocumentUpdate) o; + if (!(o instanceof DocumentUpdate that)) return false; if (docId != null ? !docId.equals(that.docId) : that.docId != null) return false; if (documentType != null ? !documentType.equals(that.documentType) : that.documentType != null) return false; @@ -413,7 +411,7 @@ public class DocumentUpdate extends DocumentOperation implements Iterable<FieldP } /** - * Returns whether or not this field update contains any field- or field path updates. + * Returns whether this field update contains any field- or field path updates. * * @return True if this update is empty. */ diff --git a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java index 8b4b94f6bbf..d09967f973f 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/StringFieldValue.java @@ -29,6 +29,9 @@ import java.util.Objects; */ public class StringFieldValue extends FieldValue { + // TODO: remove this, it's a temporary workaround for invalid data stored before unicode validation was fixed + private static final boolean replaceInvalidUnicode = System.getProperty("vespa.replace_invalid_unicode", "false").equals("true"); + private static class Factory extends PrimitiveDataType.Factory { @Override public FieldValue create() { return new StringFieldValue(); } @Override public FieldValue create(String value) { return new StringFieldValue(value); } @@ -56,16 +59,17 @@ public class StringFieldValue extends FieldValue { setValue(value); } - private static void validateTextString(String value) { + private static String validateTextString(String value) { if ( ! Text.isValidTextString(value)) { - throw new IllegalArgumentException("The string field value contains illegal code point 0x" + - Integer.toHexString(Text.validateTextString(value).getAsInt()).toUpperCase()); + if (replaceInvalidUnicode) return Text.stripInvalidCharacters(value); + else throw new IllegalArgumentException("The string field value contains illegal code point 0x" + + Integer.toHexString(Text.validateTextString(value).getAsInt()).toUpperCase()); } + return value; } private void setValue(String value) { - validateTextString(value); - this.value = value; + this.value = validateTextString(value); } /** diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 7b1042903ec..ed6bdc721a0 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -347,4 +347,5 @@ public class JsonSerializationHelper { wrapIOException(() -> generator.writeFieldName(field.getName())); } } + } diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index 7e82e830064..2b0ba138466 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -7,7 +7,9 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.StreamReadConstraints; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; import com.yahoo.document.annotation.AnnotationReference; import com.yahoo.document.datatypes.Array; @@ -263,6 +265,26 @@ public class JsonWriter implements DocumentWriter { // NOP, fetched from Document } + @Override + public void write(DocumentRemove documentRemove) { + try { + generator.writeStartObject(); + + serializeStringField(generator, new FieldBase("remove"), new StringFieldValue(documentRemove.getId().toString())); + + generator.writeEndObject(); + generator.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void write(DocumentUpdate documentUpdate) { + var serializer = new DocumentUpdateJsonSerializer(generator); + serializer.serialize(documentUpdate); + } + /** * Utility method to easily serialize a single document. * diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java index 0202ec8bf23..b9cc439ad54 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java @@ -17,8 +17,7 @@ import com.yahoo.document.update.TensorRemoveUpdate; /** * Interface for writing document updates in custom serializers. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.27 + * @author Einar M R Rosenvinge */ public interface DocumentUpdateWriter { void write(DocumentUpdate update); diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java index c84140c9ea0..16125926fe6 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java @@ -3,7 +3,9 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; /** * @author ravishar @@ -17,4 +19,8 @@ public interface DocumentWriter extends FieldWriter { void write(DocumentType type); + void write(DocumentRemove documentRemove); + + void write(DocumentUpdate documentUpdate); + } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index 17ab3890bcf..4cb836860be 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -8,6 +8,7 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; @@ -426,6 +427,10 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume putShort(null, (short) 0); // Used to hold the version. Is now always 0. } + public void write(DocumentRemove documentRemove) { + throw new UnsupportedOperationException("serializing remove not implemented"); + } + public void write(Annotation annotation) { buf.putInt(annotation.getType().getId()); //name hash |