From 41a5368c200e5417c10d5f9bba9b21e7f18c00e6 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Mon, 11 Mar 2024 14:38:08 +0100 Subject: Support serializing document remove operation to json format --- document/abi-spec.json | 4 ++- .../document/json/JsonSerializationHelper.java | 1 + .../java/com/yahoo/document/json/JsonWriter.java | 14 +++++++++ .../document/serialization/DocumentWriter.java | 3 ++ .../serialization/VespaDocumentSerializer6.java | 5 ++++ .../com/yahoo/vespa/feed/perf/SimpleFeeder.java | 35 +++++++++++++++------- .../yahoo/vespa/feed/perf/SimpleFeederTest.java | 21 +++++++++---- 7 files changed, 66 insertions(+), 17 deletions(-) diff --git a/document/abi-spec.json b/document/abi-spec.json index ca06e2547d7..ca4b4da3ca0 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -2862,7 +2862,8 @@ "methods" : [ "public abstract void write(com.yahoo.document.Document)", "public abstract void write(com.yahoo.document.DocumentId)", - "public abstract void write(com.yahoo.document.DocumentType)" + "public abstract void write(com.yahoo.document.DocumentType)", + "public abstract void write(com.yahoo.document.DocumentRemove)" ], "fields" : [ ] }, @@ -3104,6 +3105,7 @@ "public void write(com.yahoo.vespa.objects.FieldBase, com.yahoo.document.annotation.AnnotationReference)", "public void write(com.yahoo.document.DocumentId)", "public void write(com.yahoo.document.DocumentType)", + "public void write(com.yahoo.document.DocumentRemove)", "public void write(com.yahoo.document.annotation.Annotation)", "public void write(com.yahoo.document.annotation.SpanTree)", "public void write(com.yahoo.document.annotation.SpanNode)", diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 7b1042903ec..ed6bdc721a0 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -347,4 +347,5 @@ public class JsonSerializationHelper { wrapIOException(() -> generator.writeFieldName(field.getName())); } } + } diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index 7e82e830064..9cbadb65f10 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.StreamReadConstraints; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.annotation.AnnotationReference; @@ -263,6 +264,19 @@ public class JsonWriter implements DocumentWriter { // NOP, fetched from Document } + public void write(DocumentRemove documentRemove) { + try { + generator.writeStartObject(); + + serializeStringField(generator, new FieldBase("remove"), new StringFieldValue(documentRemove.getId().toString())); + + generator.writeEndObject(); + generator.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + /** * Utility method to easily serialize a single document. * diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java index c84140c9ea0..10483d8609f 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentWriter.java @@ -3,6 +3,7 @@ package com.yahoo.document.serialization; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; /** @@ -17,4 +18,6 @@ public interface DocumentWriter extends FieldWriter { void write(DocumentType type); + void write(DocumentRemove documentRemove); + } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index 17ab3890bcf..4cb836860be 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -8,6 +8,7 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; @@ -426,6 +427,10 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume putShort(null, (short) 0); // Used to hold the version. Is now always 0. } + public void write(DocumentRemove documentRemove) { + throw new UnsupportedOperationException("serializing remove not implemented"); + } + public void write(Annotation annotation) { buf.putInt(annotation.getType().getId()); //name hash diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java index b4ca98f316f..b41bb29376b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java @@ -185,21 +185,34 @@ public class SimpleFeeder implements ReplyHandler { } } public void send(FeedOperation op) { - if (op.getType() == FeedOperation.Type.DOCUMENT) { - if (!isFirst) { - try { - outputStream.write(','); - outputStream.write('\n'); - } catch (IOException e) { - failure.set(e); - } - } else { - isFirst = false; + switch (op.getType()) { + case DOCUMENT -> { + addCommaAndNewline(); + writer.write(op.getDocumentPut().getDocument()); } - writer.write(op.getDocumentPut().getDocument()); + case REMOVE -> { + addCommaAndNewline(); + writer.write(op.getDocumentRemove()); + } + default -> { /* TODO: No more operations supported yet */ } } numReplies.incrementAndGet(); } + + private void addCommaAndNewline() { + if (! isFirst) { + try { + outputStream.write(','); + outputStream.write('\n'); + } catch (IOException e) { + failure.set(e); + } + } + else { + isFirst = false; + } + } + public void close() throws Exception { outputStream.write('\n'); outputStream.write(']'); diff --git a/vespaclient-java/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java b/vespaclient-java/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java index 582148e8eaa..28859d0e7c4 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java @@ -87,8 +87,13 @@ public class SimpleFeederTest { "", "(.+\n)+" + "\\s*\\d+,\\s*3,.+\n"); - assertEquals(58, dump.size()); - assertEquals("[\n{\"id\":\"id:simple:simple::0\",\"fields\":{\"my_str\":\"foo\"}}\n]", dump.toString()); + assertEquals(93, dump.size()); + assertEquals(""" + [ + {"id":"id:simple:simple::0","fields":{"my_str":"foo"}}, + {"remove":"id:simple:simple::2"} + ]""", + dump.toString()); } @Test @@ -116,8 +121,14 @@ public class SimpleFeederTest { "", "(.+\n)+" + "\\s*\\d+,\\s*3,.+\n"); - assertEquals(115, dump.size()); - assertEquals("[\n{\"id\":\"id:simple:simple::0\",\"fields\":{\"my_str\":\"foo\"}},\n {\"id\":\"id:simple:simple::1\",\"fields\":{\"my_str\":\"bar\"}}\n]", dump.toString()); + assertEquals(150, dump.size()); + assertEquals(""" + [ + {"id":"id:simple:simple::0","fields":{"my_str":"foo"}}, + {"id":"id:simple:simple::1","fields":{"my_str":"bar"}}, + {"remove":"id:simple:simple::2"} + ]""", + dump.toString()); assertFeed(dump.toString(), new MessageHandler() { @Override @@ -129,7 +140,7 @@ public class SimpleFeederTest { }, "", "(.+\n)+" + - "\\s*\\d+,\\s*2,.+\n"); + "\\s*\\d+,\\s*3,.+\n"); } @Test -- cgit v1.2.3