diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2019-04-25 19:57:57 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2019-04-25 19:57:57 +0200 |
commit | 907158959c159904c3f4d20274c565806746ac15 (patch) | |
tree | f4db3e3d2fc81cf061cb1811b5bc3af4ca91695f | |
parent | 249fe76c9437d0f1a033294df98d8d8101baef2c (diff) |
Refactor to allow for lazy decode.
14 files changed, 160 insertions, 171 deletions
diff --git a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java index 1b5681e7146..f773173e89a 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java @@ -9,6 +9,7 @@ import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.FeedReader; import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; @@ -34,13 +35,13 @@ public class JsonFeedReader implements FeedReader { } @Override - public void read(Operation operation) throws Exception { + public FeedOperation read() throws Exception { DocumentOperation documentOperation = reader.next(); + Operation operation = new Operation(); if (documentOperation == null) { stream.close(); - operation.setInvalid(); - return; + return operation; } if (documentOperation instanceof DocumentUpdate) { @@ -54,6 +55,7 @@ public class JsonFeedReader implements FeedReader { } operation.setCondition(documentOperation.getCondition()); + return operation; } } diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java b/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java new file mode 100644 index 00000000000..58371faeced --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java @@ -0,0 +1,40 @@ +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; + +public abstract class FeedOperation { + public enum Type {DOCUMENT, REMOVE, UPDATE, INVALID} + + private Type type; + protected FeedOperation() { + this(Type.INVALID); + } + protected FeedOperation(Type type) { + this.type = type; + } + public final Type getType() { return type; } + protected final void setType(Type type) { + this.type = type; + } + + public abstract Document getDocument(); + public abstract DocumentUpdate getDocumentUpdate(); + public abstract DocumentId getRemove(); + + public TestAndSetCondition getCondition() { + return TestAndSetCondition.NOT_PRESENT_CONDITION; + } + @Override + public String toString() { + return "Operation{" + + "type=" + getType() + + ", doc=" + getDocument() + + ", remove=" + getRemove() + + ", docUpdate=" + getDocumentUpdate() + + " testandset=" + getCondition() + + '}'; + } +}
\ No newline at end of file diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java index 2c130cae782..c993d5a5153 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespaxmlparser; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; - /** * Minimal interface for reading operations from a stream for a feeder. * @@ -14,8 +12,7 @@ public interface FeedReader { /** * Reads the next operation from the stream. - * @param operation The operation to fill in. Operation is unchanged if none was found. + * @return operation, possibly invalid if none was found. */ - void read(Operation operation) throws Exception; - + FeedOperation read() throws Exception; } diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java index 14f4fd9484c..e34f3e7e4e4 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java @@ -74,22 +74,14 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { } } - public enum OperationType { - DOCUMENT, - REMOVE, - UPDATE, - INVALID - } - /** * Represents a feed operation found by the parser. Can be one of the following types: * - getType() == DOCUMENT: getDocument() is valid. * - getType() == REMOVE: getRemove() is valid. * - getType() == UPDATE: getUpdate() is valid. */ - public static class Operation { + public static class Operation extends FeedOperation { - private OperationType type; private Document doc; private DocumentId remove; private DocumentUpdate docUpdate; @@ -100,41 +92,38 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { } public void setInvalid() { - type = OperationType.INVALID; doc = null; remove = null; docUpdate = null; condition = TestAndSetCondition.NOT_PRESENT_CONDITION; } - public OperationType getType() { - return type; - } - + @Override public Document getDocument() { return doc; } public void setDocument(Document doc) { - this.type = OperationType.DOCUMENT; + setType(Type.DOCUMENT); this.doc = doc; } + @Override public DocumentId getRemove() { return remove; } public void setRemove(DocumentId remove) { - this.type = OperationType.REMOVE; + setType(Type.REMOVE); this.remove = remove; } - + @Override public DocumentUpdate getDocumentUpdate() { return docUpdate; } public void setDocumentUpdate(DocumentUpdate docUpdate) { - this.type = OperationType.UPDATE; + setType(Type.UPDATE); this.docUpdate = docUpdate; } @@ -142,19 +131,11 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { this.condition = condition; } + @Override public TestAndSetCondition getCondition() { return condition; } - @Override - public String toString() { - return "Operation{" + - "type=" + type + - ", doc=" + doc + - ", remove=" + remove + - ", docUpdate=" + docUpdate + - '}'; - } } /** @@ -164,12 +145,11 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { * * @return The list of all read operations. */ - public List<Operation> readAll() throws Exception { - List<Operation> list = new ArrayList<>(); + public List<FeedOperation> readAll() throws Exception { + List<FeedOperation> list = new ArrayList<>(); while (true) { - Operation op = new Operation(); - read(op); - if (op.getType() == OperationType.INVALID) { + FeedOperation op = read(); + if (op.getType() == FeedOperation.Type.INVALID) { return list; } else { list.add(op); @@ -181,10 +161,9 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { * @see com.yahoo.vespaxmlparser.FeedReader#read(com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation) */ @Override - public void read(Operation operation) throws Exception { + public FeedOperation read() throws Exception { + Operation operation = new Operation(); String startTag = null; - operation.setInvalid(); - try { while (reader.hasNext()) { int type = reader.next(); @@ -197,13 +176,13 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { Document document = new Document(documentReader); operation.setDocument(document); operation.setCondition(TestAndSetCondition.fromConditionString(documentReader.getCondition())); - return; + return operation; } else if ("update".equals(startTag)) { VespaXMLUpdateReader updateReader = new VespaXMLUpdateReader(reader, docTypeManager); DocumentUpdate update = new DocumentUpdate(updateReader); operation.setDocumentUpdate(update); operation.setCondition(TestAndSetCondition.fromConditionString(updateReader.getCondition())); - return; + return operation; } else if ("remove".equals(startTag)) { boolean documentIdFound = false; @@ -224,7 +203,7 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { operation.setCondition(TestAndSetCondition.fromConditionString(condition)); - return; + return operation; } else { throw newDeserializeException("Element \"" + startTag + "\" not allowed in this context"); } @@ -243,6 +222,7 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { throw(e); } + return operation; } } diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java index a7fd782484e..e2aafcb4fdc 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java @@ -26,7 +26,7 @@ public class PositionParserTestCase { mgr.registerDocumentType(docType); VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_position.xml", mgr); - Iterator<VespaXMLFeedReader.Operation> it = parser.readAll().iterator(); + Iterator<FeedOperation> it = parser.readAll().iterator(); assertTrue(it.hasNext()); assertDocument(PositionDataType.valueOf(1, 2), it.next()); assertTrue(it.hasNext()); @@ -38,9 +38,9 @@ public class PositionParserTestCase { assertFalse(it.hasNext()); } - private static void assertDocument(Struct expected, VespaXMLFeedReader.Operation operation) { + private static void assertDocument(Struct expected, FeedOperation operation) { assertNotNull(operation); - assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, operation.getType()); + assertEquals(FeedOperation.Type.DOCUMENT, operation.getType()); Document doc = operation.getDocument(); assertNotNull(doc); assertEquals(expected, doc.getFieldValue("my_pos")); diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java index dcdea0975ad..0ccae4dbde5 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java @@ -28,7 +28,7 @@ public class UriParserTestCase { mgr.registerDocumentType(docType); VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_uri.xml", mgr); - Iterator<VespaXMLFeedReader.Operation> it = parser.readAll().iterator(); + Iterator<FeedOperation> it = parser.readAll().iterator(); Document doc = nextDocument(it); assertNotNull(doc); @@ -59,21 +59,21 @@ public class UriParserTestCase { assertFalse(it.hasNext()); } - private static Document nextDocument(Iterator<VespaXMLFeedReader.Operation> it) { + private static Document nextDocument(Iterator<FeedOperation> it) { assertTrue(it.hasNext()); - VespaXMLFeedReader.Operation op = it.next(); + FeedOperation op = it.next(); assertNotNull(op); - assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, op.getType()); + assertEquals(FeedOperation.Type.DOCUMENT, op.getType()); Document doc = op.getDocument(); assertNotNull(doc); return doc; } - private static DocumentUpdate nextUpdate(Iterator<VespaXMLFeedReader.Operation> it) { + private static DocumentUpdate nextUpdate(Iterator<FeedOperation> it) { assertTrue(it.hasNext()); - VespaXMLFeedReader.Operation op = it.next(); + FeedOperation op = it.next(); assertNotNull(op); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate upd = op.getDocumentUpdate(); assertNotNull(upd); return upd; diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java index 29567177642..e33dbfe8898 100755 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java @@ -40,7 +40,7 @@ public class VespaXMLReaderTestCase { } @Test - public void testMapNoKey() throws Exception { + public void testMapNoKey() { try { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testmapnokey.xml", manager); parser.readAll(); @@ -51,7 +51,7 @@ public class VespaXMLReaderTestCase { } @Test - public void testMapNoValue() throws Exception { + public void testMapNoValue() { try { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testmapnovalue.xml", manager); parser.readAll(); @@ -64,10 +64,9 @@ public class VespaXMLReaderTestCase { @Test public void testNews1() throws Exception { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testalltypes.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertTrue(VespaXMLFeedReader.OperationType.INVALID != op.getType()); + assertTrue(FeedOperation.Type.INVALID != op.getType()); Document doc = op.getDocument(); assertEquals(new StringFieldValue("testUrl"), doc.getFieldValue("url")); assertEquals(new StringFieldValue("testTitle"), doc.getFieldValue("title")); @@ -149,10 +148,9 @@ public class VespaXMLReaderTestCase { public void testNews3() throws Exception { // Updating all elements in a documentType VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test03.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -214,10 +212,9 @@ public class VespaXMLReaderTestCase { // Test on adding just a few fields to a DocumentUpdate (implies other fields to null) VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test04.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); //url @@ -269,10 +266,9 @@ public class VespaXMLReaderTestCase { // Adding a few new fields to a Document using different syntax VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test05.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -334,20 +330,19 @@ public class VespaXMLReaderTestCase { // long value with txt try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); } // empty string - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); assertEquals("doc:news:http://news6b", op.getDocument().getId().toString()); // int array with text try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -355,7 +350,7 @@ public class VespaXMLReaderTestCase { // long array with whitespace try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -363,7 +358,7 @@ public class VespaXMLReaderTestCase { // byte array with value try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -371,7 +366,7 @@ public class VespaXMLReaderTestCase { // float array with string try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -379,7 +374,7 @@ public class VespaXMLReaderTestCase { // weighted set of int with string try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -387,7 +382,7 @@ public class VespaXMLReaderTestCase { // weighted set of int with string as weight try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -395,17 +390,17 @@ public class VespaXMLReaderTestCase { // weighted set of string with string as weight try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); } - parser.read(op = new VespaXMLFeedReader.Operation()); + op = parser.read(); assertEquals("doc:news:http://news6j", op.getDocument().getId().toString()); - parser.read(op = new VespaXMLFeedReader.Operation()); - assertEquals(VespaXMLFeedReader.OperationType.INVALID, op.getType()); + op = parser.read(); + assertEquals(FeedOperation.Type.INVALID, op.getType()); } @Test @@ -414,10 +409,9 @@ public class VespaXMLReaderTestCase { // are also some updates that will fail (be skipped). VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test07.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -449,7 +443,7 @@ public class VespaXMLReaderTestCase { // Trying arithmetic on string (b) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -457,7 +451,7 @@ public class VespaXMLReaderTestCase { // "By" as string (c) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -465,7 +459,7 @@ public class VespaXMLReaderTestCase { // Empty key in weighted set of int (d) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -473,7 +467,7 @@ public class VespaXMLReaderTestCase { // No "by" attribute (e) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -481,7 +475,7 @@ public class VespaXMLReaderTestCase { // Float key as string (f) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -492,10 +486,9 @@ public class VespaXMLReaderTestCase { public void testNews8() throws Exception { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test08.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -518,24 +511,21 @@ public class VespaXMLReaderTestCase { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test09.xml", manager); { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.REMOVE, op.getType()); + assertEquals(FeedOperation.Type.REMOVE, op.getType()); assertEquals("doc:news:http://news9a", op.getRemove().toString()); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.REMOVE, op.getType()); + assertEquals(FeedOperation.Type.REMOVE, op.getType()); assertEquals("doc:news:http://news9b", op.getRemove().toString()); } { // Remove without documentid. Not supported. - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); try { - parser.read(op); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -547,8 +537,7 @@ public class VespaXMLReaderTestCase { public void testNews10() throws Exception { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test10.xml", manager); { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); Document doc = op.getDocument(); assertEquals(new StringFieldValue("testUrl"), doc.getFieldValue("url")); @@ -585,15 +574,13 @@ public class VespaXMLReaderTestCase { assertEquals(Integer.valueOf(14), strWset.get(new StringFieldValue("string14"))); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); Document doc = op.getDocument(); assertNotNull(doc); assertEquals(new StringFieldValue("testUrl2"), doc.getFieldValue("url")); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); DocumentUpdate upd = op.getDocumentUpdate(); assertNull(upd.getFieldUpdate("url")); @@ -629,8 +616,7 @@ public class VespaXMLReaderTestCase { .getWeight()); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); DocumentUpdate upd = op.getDocumentUpdate(); assertEquals(new StringFieldValue("assignUrl"), @@ -661,15 +647,13 @@ public class VespaXMLReaderTestCase { assertNull(upd.getFieldUpdate("weightedsetstring")); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); assertEquals("doc:news:http://news10e", op.getRemove().toString()); } { // Illegal remove without documentid attribute try { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -682,10 +666,9 @@ public class VespaXMLReaderTestCase { // Adding a few new fields to a Document using different syntax VespaXMLFeedReader parser = new VespaXMLFeedReader("src/tests/vespaxml/fieldpathupdates.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -801,13 +784,13 @@ public class VespaXMLReaderTestCase { .configure(m, "file:src/test/java/com/yahoo/document/documentmanager.docindoc.cfg"); VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_docindoc.xml", m); - List<VespaXMLFeedReader.Operation> ops = parser.readAll(); + List<FeedOperation> ops = parser.readAll(); assertEquals(1, ops.size()); - VespaXMLFeedReader.Operation op = ops.get(0); + FeedOperation op = ops.get(0); System.err.println(op); - assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, op.getType()); + assertEquals(FeedOperation.Type.DOCUMENT, op.getType()); assertNull(op.getRemove()); assertNull(op.getDocumentUpdate()); assertNotNull(op.getDocument()); @@ -886,8 +869,7 @@ public class VespaXMLReaderTestCase { final int NUM_OPERATIONS_IN_FEED = 3; for (int i = 0; i < NUM_OPERATIONS_IN_FEED; i++) { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); assertTrue("Missing test and set condition", op.getCondition().isPresent()); assertEquals("Condition is not the same as in xml feed", "news.value_long == 1", op.getCondition().getSelection()); diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java index 1fdbd2db9c0..9e32033634a 100644 --- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java +++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java @@ -31,6 +31,7 @@ import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.routing.Route; import com.yahoo.vespaxmlparser.FeedReader; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import net.jpountz.xxhash.XXHashFactory; @@ -75,7 +76,7 @@ public class SimpleFeeder implements ReplyHandler { } private interface Destination { - void send(VespaXMLFeedReader.Operation op); + void send(FeedOperation op); void close() throws Exception; } @@ -90,7 +91,7 @@ public class SimpleFeeder implements ReplyHandler { this.session = session; this.failure = failure; } - public void send(VespaXMLFeedReader.Operation op) { + public void send(FeedOperation op) { Message msg = newMessage(op); if (msg == null) { err.println("ignoring operation; " + op.getType()); @@ -128,8 +129,8 @@ public class SimpleFeeder implements ReplyHandler { failure.set(e); } } - public void send(VespaXMLFeedReader.Operation op) { - if (op.getType() == VespaXMLFeedReader.OperationType.DOCUMENT) { + public void send(FeedOperation op) { + if (op.getType() == FeedOperation.Type.DOCUMENT) { if (!isFirst) { try { outputStream.write(','); @@ -173,16 +174,16 @@ public class SimpleFeeder implements ReplyHandler { failure.set(e); } } - public void send(VespaXMLFeedReader.Operation op) { + public void send(FeedOperation op) { DocumentSerializer writer = DocumentSerializerFactory.createHead(buffer); int type = NONE; - if (op.getType() == VespaXMLFeedReader.OperationType.DOCUMENT) { + if (op.getType() == FeedOperation.Type.DOCUMENT) { writer.write(op.getDocument()); type = DOCUMENT; - } else if (op.getType() == VespaXMLFeedReader.OperationType.UPDATE) { + } else if (op.getType() == FeedOperation.Type.UPDATE) { writer.write(op.getDocumentUpdate()); type = UPDATE; - } else if (op.getType() == VespaXMLFeedReader.OperationType.REMOVE) { + } else if (op.getType() == FeedOperation.Type.REMOVE) { writer.write(op.getRemove()); type = REMOVE; } @@ -224,11 +225,11 @@ public class SimpleFeeder implements ReplyHandler { } } @Override - public void read(VespaXMLFeedReader.Operation operation) throws Exception { + public FeedOperation read() throws Exception { + VespaXMLFeedReader.Operation operation = new VespaXMLFeedReader.Operation(); int read = in.read(prefix); if (read != prefix.length) { - operation.setInvalid(); - return; + return operation; } ByteBuffer header = ByteBuffer.wrap(prefix); int sz = header.getInt(); @@ -253,6 +254,7 @@ public class SimpleFeeder implements ReplyHandler { } else { throw new IllegalArgumentException("Unknown operation " + type); } + return operation; } } @@ -274,7 +276,7 @@ public class SimpleFeeder implements ReplyHandler { : new MbusDestination(session, params.getRoute(), failure, params.getStdErr()); } - private void sendOperation(VespaXMLFeedReader.Operation op) { + private void sendOperation(FeedOperation op) { destination.send(op); } @@ -308,9 +310,8 @@ public class SimpleFeeder implements ReplyHandler { printHeader(); long numMessages = 0; while (failure.get() == null) { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - reader.read(op); - if (op.getType() == VespaXMLFeedReader.OperationType.INVALID) { + FeedOperation op = reader.read(); + if (op.getType() == FeedOperation.Type.INVALID) { break; } if (executor != null) { @@ -335,7 +336,7 @@ public class SimpleFeeder implements ReplyHandler { mbus.destroy(); } - private static Message newMessage(VespaXMLFeedReader.Operation op) { + private static Message newMessage(FeedOperation op) { switch (op.getType()) { case DOCUMENT: { PutDocumentMessage message = new PutDocumentMessage(new DocumentPut(op.getDocument())); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index ac2c9515d71..947fcb637fb 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -17,6 +17,7 @@ import com.yahoo.net.HostName; import com.yahoo.vespa.http.client.core.ErrorCode; import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import com.yahoo.yolean.Exceptions; @@ -274,7 +275,7 @@ class ClientFeederV3 { /** Returns the next message in the stream, or null if none */ protected DocumentOperationMessageV3 getNextMessage( String operationId, InputStream requestInputStream, FeederSettings settings) throws Exception { - VespaXMLFeedReader.Operation operation = streamReaderV3.getNextOperation(requestInputStream, settings); + FeedOperation operation = streamReaderV3.getNextOperation(requestInputStream, settings); // This is a bit hard to set up while testing, so we accept that things are not perfect. if (sourceSession.getResource().session() != null) { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java index ca2fdd6b329..6df6bba5a59 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java @@ -9,11 +9,7 @@ import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.jdisc.Metric; import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.routing.ErrorDirective; -import com.yahoo.messagebus.routing.Hop; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; -import com.yahoo.yolean.Exceptions; +import com.yahoo.vespaxmlparser.FeedOperation; /** * Keeps an operation with its message. @@ -40,18 +36,7 @@ class DocumentOperationMessageV3 { return operationId; } - static DocumentOperationMessageV3 newErrorMessage(String operationId, Exception exception) { - Message feedErrorMessageV3 = new FeedErrorMessage(operationId); - DocumentOperationMessageV3 msg = new DocumentOperationMessageV3(operationId, feedErrorMessageV3); - Hop hop = new Hop(); - hop.addDirective(new ErrorDirective(Exceptions.toMessageString(exception))); - Route route = new Route(); - route.addHop(hop); - feedErrorMessageV3.setRoute(route); - return msg; - } - - static DocumentOperationMessageV3 newUpdateMessage(VespaXMLFeedReader.Operation op, String operationId) { + private static DocumentOperationMessageV3 newUpdateMessage(FeedOperation op, String operationId) { DocumentUpdate update = op.getDocumentUpdate(); update.setCondition(op.getCondition()); Message msg = new UpdateDocumentMessage(update); @@ -60,7 +45,7 @@ class DocumentOperationMessageV3 { return new DocumentOperationMessageV3(id, msg); } - static DocumentOperationMessageV3 newRemoveMessage(VespaXMLFeedReader.Operation op, String operationId) { + static DocumentOperationMessageV3 newRemoveMessage(FeedOperation op, String operationId) { DocumentRemove remove = new DocumentRemove(op.getRemove()); remove.setCondition(op.getCondition()); Message msg = new RemoveDocumentMessage(remove); @@ -69,7 +54,7 @@ class DocumentOperationMessageV3 { return new DocumentOperationMessageV3(id, msg); } - static DocumentOperationMessageV3 newPutMessage(VespaXMLFeedReader.Operation op, String operationId) { + private static DocumentOperationMessageV3 newPutMessage(FeedOperation op, String operationId) { DocumentPut put = new DocumentPut(op.getDocument()); put.setCondition(op.getCondition()); Message msg = new PutDocumentMessage(put); @@ -78,7 +63,7 @@ class DocumentOperationMessageV3 { return new DocumentOperationMessageV3(id, msg); } - static DocumentOperationMessageV3 create(VespaXMLFeedReader.Operation operation, String operationId, Metric metric) { + static DocumentOperationMessageV3 create(FeedOperation operation, String operationId, Metric metric) { switch (operation.getType()) { case DOCUMENT: metric.add(MetricNames.NUM_PUTS, 1, null /*metricContext*/); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java index 00290c4fb09..69f810ad4c7 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java @@ -5,8 +5,8 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.document.DocumentTypeManager; import com.yahoo.vespa.http.client.core.Encoder; import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.FeedReader; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import java.io.IOException; import java.io.InputStream; @@ -30,15 +30,14 @@ public class StreamReaderV3 { this.docTypeManager = docTypeManager; } - public VespaXMLFeedReader.Operation getNextOperation( - InputStream requestInputStream, FeederSettings settings) throws Exception { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); + public FeedOperation getNextOperation(InputStream requestInputStream, FeederSettings settings) throws Exception { + FeedOperation op = null; int length = readByteLength(requestInputStream); try (InputStream limitedInputStream = new ByteLimitedInputStream(requestInputStream, length)){ FeedReader reader = feedReaderFactory.createReader(limitedInputStream, docTypeManager, settings.dataFormat); - reader.read(op); + op = reader.read(); } return op; } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java index b399b1197da..606d21c0059 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java @@ -45,9 +45,10 @@ public class MockReader implements FeedReader { } @Override - public void read(Operation operation) throws Exception { + public FeedOperation read() throws Exception { + Operation operation = new Operation(); if (finished) { - return; + return operation; } byte whatToDo = stream.getNextOperation(); @@ -70,6 +71,7 @@ public class MockReader implements FeedReader { case 4: throw new RuntimeException("boom"); } + return operation; } } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java index e354cba141d..a2bb2e8514b 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java @@ -10,6 +10,7 @@ import java.util.List; import javax.xml.stream.XMLStreamException; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.FeedReader; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; @@ -80,14 +81,13 @@ public abstract class Feeder { while (!sender.isAborted()) { try { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - reader.read(op); + FeedOperation op = reader.read(); if (createIfNonExistent && op.getDocumentUpdate() != null) { op.getDocumentUpdate().setCreateIfNonExistent(true); } // Done feeding. - if (op.getType() == VespaXMLFeedReader.OperationType.INVALID) { + if (op.getType() == FeedOperation.Type.INVALID) { break; } else { sender.sendOperation(op); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java index b441e81a829..a1666a83856 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedapi; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; +import com.yahoo.vespaxmlparser.FeedOperation; /** * Wrapper class for SimpleFeedAccess to send various XML operations. @@ -18,7 +18,7 @@ public class VespaFeedSender { return sender.isAborted(); } - public void sendOperation(VespaXMLFeedReader.Operation op) { + public void sendOperation(FeedOperation op) { switch (op.getType()) { case DOCUMENT: sender.put(op.getDocument(), op.getCondition()); |