diff options
Diffstat (limited to 'docproc/src/test')
46 files changed, 3389 insertions, 0 deletions
diff --git a/docproc/src/test/cfg/docproc-chain-arguments.cfg b/docproc/src/test/cfg/docproc-chain-arguments.cfg new file mode 100644 index 00000000000..bbd66d7591d --- /dev/null +++ b/docproc/src/test/cfg/docproc-chain-arguments.cfg @@ -0,0 +1,20 @@ +docprocchain[1] +docprocchain[0].name "chain-arguments" +docprocchain[0].processor[3] + +docprocchain[0].processor[0].classname "com.yahoo.docproc.configuration.test.DocprocServiceArgumentConfigurationTestCase$TestDocumentProcessor1" +docprocchain[0].processor[0].argument[2] +docprocchain[0].processor[0].argument[0].name arg1 +docprocchain[0].processor[0].argument[0].value val1 +docprocchain[0].processor[0].argument[1].name arg2 +docprocchain[0].processor[0].argument[1].value val2 + +docprocchain[0].processor[1].classname "com.yahoo.docproc.configuration.test.DocprocServiceArgumentConfigurationTestCase$TestDocumentProcessor2" +docprocchain[0].processor[1].argument[1] +docprocchain[0].processor[1].argument[0].name arg1 +docprocchain[0].processor[1].argument[0].value val3 + +docprocchain[0].processor[2].classname "com.yahoo.docproc.configuration.test.DocprocServiceArgumentConfigurationTestCase$TestDocumentProcessor3" +docprocchain[0].processor[2].argument[1] +docprocchain[0].processor[2].argument[0].name arg2 +docprocchain[0].processor[2].argument[0].value val4 diff --git a/docproc/src/test/cfg/docproc-chain-empty.cfg b/docproc/src/test/cfg/docproc-chain-empty.cfg new file mode 100644 index 00000000000..091ed109913 --- /dev/null +++ b/docproc/src/test/cfg/docproc-chain-empty.cfg @@ -0,0 +1,6 @@ +docprocchain[2] +docprocchain[0].name "baloo" +docprocchain[0].processor[1] +docprocchain[0].processor[0].classname "com.yahoo.docproc.test.DocumentProcessingAbstractTestCase$TestDocumentProcessor1" +docprocchain[1].name "badoo" +docprocchain[1].processor[0] diff --git a/docproc/src/test/cfg/docproc-chain-parameters.cfg b/docproc/src/test/cfg/docproc-chain-parameters.cfg new file mode 100644 index 00000000000..0780030bd3b --- /dev/null +++ b/docproc/src/test/cfg/docproc-chain-parameters.cfg @@ -0,0 +1,24 @@ +docprocchain[1] +docprocchain[0].name "parameters" +docprocchain[0].processor[3] + +docprocchain[0].processor[0].classname "com.yahoo.docproc.configuration.test.DocprocServiceParametersConfigurationTestCase$TestDocumentProcessor1" +docprocchain[0].processor[0].parameter[4] +docprocchain[0].processor[0].parameter[0].name param1 +docprocchain[0].processor[0].parameter[0].value val1 +docprocchain[0].processor[0].parameter[1].name param1 +docprocchain[0].processor[0].parameter[1].value val2 +docprocchain[0].processor[0].parameter[2].name param1 +docprocchain[0].processor[0].parameter[2].value val3 +docprocchain[0].processor[0].parameter[3].name param2 +docprocchain[0].processor[0].parameter[3].value val0 + +docprocchain[0].processor[1].classname "com.yahoo.docproc.configuration.test.DocprocServiceParametersConfigurationTestCase$TestDocumentProcessor2" +docprocchain[0].processor[1].parameter[1] +docprocchain[0].processor[1].parameter[0].name param1 +docprocchain[0].processor[1].parameter[0].value val3 + +docprocchain[0].processor[2].classname "com.yahoo.docproc.configuration.test.DocprocServiceParametersConfigurationTestCase$TestDocumentProcessor3" +docprocchain[0].processor[2].parameter[1] +docprocchain[0].processor[2].parameter[0].name param2 +docprocchain[0].processor[2].parameter[0].value val4 diff --git a/docproc/src/test/cfg/docproc-chain-slow.cfg b/docproc/src/test/cfg/docproc-chain-slow.cfg new file mode 100644 index 00000000000..27cedd9407c --- /dev/null +++ b/docproc/src/test/cfg/docproc-chain-slow.cfg @@ -0,0 +1,4 @@ +docprocchain[1] +docprocchain[0].name "slow" +docprocchain[0].processor[1] +docprocchain[0].processor[0].classname "com.yahoo.docproc.configuration.test.DocprocServiceConfigurationTestCase$SlowDocumentProcessor" diff --git a/docproc/src/test/cfg/docproc-chain.cfg b/docproc/src/test/cfg/docproc-chain.cfg new file mode 100644 index 00000000000..ed40e889004 --- /dev/null +++ b/docproc/src/test/cfg/docproc-chain.cfg @@ -0,0 +1,5 @@ +docprocchain[1] +docprocchain[0].processor[3] +docprocchain[0].processor[0].classname "com.yahoo.docproc.test.DocumentProcessingAbstractTestCase$TestDocumentProcessor1" +docprocchain[0].processor[1].classname "com.yahoo.docproc.test.DocumentProcessingAbstractTestCase$TestDocumentProcessor2" +docprocchain[0].processor[2].classname "com.yahoo.docproc.test.DocumentProcessingAbstractTestCase$TestDocumentProcessor3" diff --git a/docproc/src/test/cfg/invalid/docproc-chain-nomapconstructor.cfg b/docproc/src/test/cfg/invalid/docproc-chain-nomapconstructor.cfg new file mode 100644 index 00000000000..21b6b2e4e57 --- /dev/null +++ b/docproc/src/test/cfg/invalid/docproc-chain-nomapconstructor.cfg @@ -0,0 +1,7 @@ +docprocchain[1] +docprocchain[0].name "nomapconstructor" +docprocchain[0].processor[1] +docprocchain[0].processor[0].classname "com.yahoo.docproc.configuration.test.InvalidChainTestCase$NoMapConstructorDocumentProcessor" +docprocchain[0].processor[0].parameter[1] +docprocchain[0].processor[0].parameter[0].name "dis" +docprocchain[0].processor[0].parameter[0].value "dat" diff --git a/docproc/src/test/cfg/invalid/docproc-chain-nopublicconstructor.cfg b/docproc/src/test/cfg/invalid/docproc-chain-nopublicconstructor.cfg new file mode 100644 index 00000000000..c710a4726a7 --- /dev/null +++ b/docproc/src/test/cfg/invalid/docproc-chain-nopublicconstructor.cfg @@ -0,0 +1,4 @@ +docprocchain[1] +docprocchain[0].name "nopublicconstructor" +docprocchain[0].processor[1] +docprocchain[0].processor[0].classname "com.yahoo.docproc.configuration.test.InvalidChainTestCase$NoPublicConstructorDocumentProcessor" diff --git a/docproc/src/test/cfg/invalid/docproc-chain-nostringconstructor.cfg b/docproc/src/test/cfg/invalid/docproc-chain-nostringconstructor.cfg new file mode 100644 index 00000000000..002210daf1f --- /dev/null +++ b/docproc/src/test/cfg/invalid/docproc-chain-nostringconstructor.cfg @@ -0,0 +1,4 @@ +docprocchain[1] +docprocchain[0].name "nostringconstructor" +docprocchain[0].processor[1] +docprocchain[0].processor[0].classname "com.yahoo.docproc.configuration.test.InvalidChainTestCase$NoStringConstructorDocumentProcessor" diff --git a/docproc/src/test/cfg/invalid/docproc-chain-ok.cfg b/docproc/src/test/cfg/invalid/docproc-chain-ok.cfg new file mode 100644 index 00000000000..b6a2a71d87f --- /dev/null +++ b/docproc/src/test/cfg/invalid/docproc-chain-ok.cfg @@ -0,0 +1,4 @@ +docprocchain[1] +docprocchain[0].name "ok" +docprocchain[0].processor[1] +docprocchain[0].processor[0].classname "com.yahoo.docproc.configuration.test.InvalidChainTestCase$OkDocumentProcessor" diff --git a/docproc/src/test/cfg/invalid/docproc-chain-throwingexceptioninconstructor.cfg b/docproc/src/test/cfg/invalid/docproc-chain-throwingexceptioninconstructor.cfg new file mode 100644 index 00000000000..a8296c14d20 --- /dev/null +++ b/docproc/src/test/cfg/invalid/docproc-chain-throwingexceptioninconstructor.cfg @@ -0,0 +1,4 @@ +docprocchain[1] +docprocchain[0].name "throwingexcaptioninconstructor" +docprocchain[0].processor[1] +docprocchain[0].processor[0].classname "com.yahoo.docproc.configuration.test.InvalidChainTestCase$ThrowingExceptionInConstructorDocumentProcessor" diff --git a/docproc/src/test/cfg/messagebus/docproc-chain.cfg b/docproc/src/test/cfg/messagebus/docproc-chain.cfg new file mode 100644 index 00000000000..8496ebfb05c --- /dev/null +++ b/docproc/src/test/cfg/messagebus/docproc-chain.cfg @@ -0,0 +1,2 @@ +docprocchain[1] +docprocchain[0].processor[0] diff --git a/docproc/src/test/cfg/messagebus/docproc.cfg b/docproc/src/test/cfg/messagebus/docproc.cfg new file mode 100644 index 00000000000..ef67a6993b0 --- /dev/null +++ b/docproc/src/test/cfg/messagebus/docproc.cfg @@ -0,0 +1,4 @@ +rpcserver.enabled false +rpcserver.port 6985 +slobrok.enabled false +slobrok.servicename "dpcluster.1/docproc/78" diff --git a/docproc/src/test/cfg/messagebus/documentmanager.cfg b/docproc/src/test/cfg/messagebus/documentmanager.cfg new file mode 100644 index 00000000000..36cf88c8384 --- /dev/null +++ b/docproc/src/test/cfg/messagebus/documentmanager.cfg @@ -0,0 +1,36 @@ +datatype[3] +datatype[0].id 306916075 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name test.header +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].field[3] +datatype[0].structtype[0].field[0].name test +datatype[0].structtype[0].field[0].id[0] +datatype[0].structtype[0].field[0].datatype 2 +datatype[0].structtype[0].field[1].name touched +datatype[0].structtype[0].field[1].id[0] +datatype[0].structtype[0].field[1].datatype 2 +datatype[0].structtype[0].field[2].name docno +datatype[0].structtype[0].field[2].id[0] +datatype[0].structtype[0].field[2].datatype 0 +datatype[0].documenttype[0] +datatype[1].id -1270491200 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name test.body +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[0] +datatype[1].documenttype[0] +datatype[2].id -877171244 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[1] +datatype[2].documenttype[0].name test +datatype[2].documenttype[0].version 0 +datatype[2].documenttype[0].inherits[0] +datatype[2].documenttype[0].headerstruct 306916075 +datatype[2].documenttype[0].bodystruct -1270491200 diff --git a/docproc/src/test/cfg/server/docproc-chain.cfg b/docproc/src/test/cfg/server/docproc-chain.cfg new file mode 100644 index 00000000000..0c7b14eb657 --- /dev/null +++ b/docproc/src/test/cfg/server/docproc-chain.cfg @@ -0,0 +1,3 @@ +docprocchain[1] +docprocchain[0].processor[1] +docprocchain[0].processor[0].classname "com.yahoo.docproc.server.test.ServerTestCase$BalooDocumentProcessor" diff --git a/docproc/src/test/cfg/server/docproc.cfg b/docproc/src/test/cfg/server/docproc.cfg new file mode 100644 index 00000000000..602f6fc4f10 --- /dev/null +++ b/docproc/src/test/cfg/server/docproc.cfg @@ -0,0 +1,4 @@ +rpcserver.enabled true +rpcserver.port 6985 +slobrok.enabled false +slobrok.servicename dpcluster.1/docproc/78 diff --git a/docproc/src/test/cfg/server/documentmanager.cfg b/docproc/src/test/cfg/server/documentmanager.cfg new file mode 100644 index 00000000000..b3e360e961f --- /dev/null +++ b/docproc/src/test/cfg/server/documentmanager.cfg @@ -0,0 +1,36 @@ +datatype[3] +datatype[0].id 306916075 +datatype[0].arraytype[0] +datatype[0].weightedsettype[0] +datatype[0].structtype[1] +datatype[0].structtype[0].name test.header +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].field[2] +datatype[0].structtype[0].field[0].name test +datatype[0].structtype[0].field[0].id[0] +datatype[0].structtype[0].field[0].datatype 2 +datatype[0].structtype[0].field[1].name touched +datatype[0].structtype[0].field[1].id[0] +datatype[0].structtype[0].field[1].datatype 2 +datatype[0].documenttype[0] +datatype[1].id -1270491200 +datatype[1].arraytype[0] +datatype[1].weightedsettype[0] +datatype[1].structtype[1] +datatype[1].structtype[0].name test.body +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[1] +datatype[1].structtype[0].field[0].name docno +datatype[1].structtype[0].field[0].id[0] +datatype[1].structtype[0].field[0].datatype 0 +datatype[1].documenttype[0] +datatype[2].id -877171244 +datatype[2].arraytype[0] +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[1] +datatype[2].documenttype[0].name test +datatype[2].documenttype[0].version 0 +datatype[2].documenttype[0].inherits[0] +datatype[2].documenttype[0].headerstruct 306916075 +datatype[2].documenttype[0].bodystruct -1270491200 diff --git a/docproc/src/test/java/com/yahoo/docproc/AccessesAnnotationTestCase.java b/docproc/src/test/java/com/yahoo/docproc/AccessesAnnotationTestCase.java new file mode 100644 index 00000000000..fedc0eb21f5 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/AccessesAnnotationTestCase.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class AccessesAnnotationTestCase { + + @Test + public void requireThatFieldsAreRestricted() { + DocumentType type = new DocumentType("album"); + type.addField("title", DataType.STRING); + type.addField("artist", DataType.STRING); + type.addField("year", DataType.INT); + Document doc = new Document(type, new DocumentId("doc:map:test:1")); + + MyDocProc docProc = new MyDocProc(); + DocumentPut put = new DocumentPut(doc); + Document proxy = new Call(docProc).configDoc(docProc, put).getDocument(); + proxy.setFieldValue("title", new StringFieldValue("foo")); + try { + proxy.setFieldValue("year", new IntegerFieldValue(69)); + fail("Should have failed"); + } catch (Exception e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().matches(".*not allowed.*")); + } + + proxy.getFieldValue("title"); + try { + proxy.getFieldValue("year"); + fail("Should have failed"); + } catch (Exception e) { + System.out.println(e.getMessage()); + assertTrue(e.getMessage().matches(".*not allowed.*")); + } + } + + @Accesses({ + @Accesses.Field(name = "title", dataType = "string", description = "What is done on field title", + annotations = @Accesses.Field.Tree(produces = { "sentences" })), + @Accesses.Field(name = "artist", dataType = "string", description = "What is done on field artist", + annotations = { + @Accesses.Field.Tree(name = "root", produces = { "sentences" }), + @Accesses.Field.Tree(name = "root2", consumes = { "places" }) + }), + @Accesses.Field(name = "track", dataType = "string", description = "What is done on field track") + }) + class MyDocProc extends DocumentProcessor { + + @Override + public Progress process(Processing processing) { + return null; + } + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/CallStackTestCase.java b/docproc/src/test/java/com/yahoo/docproc/CallStackTestCase.java new file mode 100644 index 00000000000..8ef832faf31 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/CallStackTestCase.java @@ -0,0 +1,291 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import java.util.Iterator; + +/** + * Tests call stacks + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class CallStackTestCase extends junit.framework.TestCase { + + private CallStack callStack, insertStack; + + private Call call1, call2, call3, noCall, newCall, insert1, insert2, insert3; + + private DocumentProcessor processor2, noProcessor; + + public CallStackTestCase(String name) { + super(name); + } + + protected void setUp() { + callStack = new CallStack(); + call1 = new Call(new TestDocumentProcessor()); + processor2 = new TestDocumentProcessor(); + call2 = new Call(processor2); + call3 = new Call(new TestDocumentProcessor()); + callStack.addLast(call1).addLast(call2).addLast(call3); + noProcessor = new TestDocumentProcessor(); + noCall = new Call(noProcessor); + newCall = new Call(new TestDocumentProcessor()); + + insert1 = new Call(new TestDocumentProcessor()); + insert2 = new Call(new TestDocumentProcessor()); + insert3 = new Call(new TestDocumentProcessor()); + insertStack = new CallStack(); + insertStack.addLast(insert1).addLast(insert2).addLast(insert3); + } + + public void testFind() { + assertSame(call2, callStack.findCall(processor2)); + assertSame(call2, callStack.findCall(processor2.getId())); + assertNull(callStack.findCall(noProcessor)); + assertNull(callStack.findCall(noProcessor.getId())); + assertNull(callStack.findCall(new TestDocumentProcessor())); + } + + public void testAddBefore() { + callStack.addBefore(call2, newCall); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(newCall, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddStackBefore() { + callStack.addBefore(call2, insertStack); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(insert1, i.next()); + assertEquals(insert2, i.next()); + assertEquals(insert3, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddAfter() { + callStack.addAfter(call2, newCall); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(newCall, i.next()); + assertEquals(call3, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddStackAfter() { + callStack.addAfter(call2, insertStack); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(insert1, i.next()); + assertEquals(insert2, i.next()); + assertEquals(insert3, i.next()); + assertEquals(call3, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddBeforeFirst() { + callStack.addBefore(call1, newCall); + Iterator i = callStack.iterator(); + assertEquals(newCall, i.next()); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddStackBeforeFirst() { + callStack.addBefore(call1, insertStack); + Iterator i = callStack.iterator(); + assertEquals(insert1, i.next()); + assertEquals(insert2, i.next()); + assertEquals(insert3, i.next()); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddAfterLast() { + callStack.addAfter(call3, newCall); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertEquals(newCall, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddStackAfterLast() { + callStack.addAfter(call3, insertStack); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertEquals(insert1, i.next()); + assertEquals(insert2, i.next()); + assertEquals(insert3, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddBeforeNonExisting() { + callStack.addBefore(noCall, newCall); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertEquals(newCall, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddStackBeforeNonExisting() { + callStack.addBefore(noCall, insertStack); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertEquals(insert1, i.next()); + assertEquals(insert2, i.next()); + assertEquals(insert3, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddAfterNonExisting() { + callStack.addAfter(noCall, newCall); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertEquals(newCall, i.next()); + assertFalse(i.hasNext()); + } + + public void testAddStackAfterNonExisting() { + callStack.addAfter(noCall, insertStack); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertEquals(insert1, i.next()); + assertEquals(insert2, i.next()); + assertEquals(insert3, i.next()); + assertFalse(i.hasNext()); + } + + public void testRemove() { + callStack.remove(call1); + Iterator i = callStack.iterator(); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertFalse(i.hasNext()); + } + + public void testRemoveNonExisting() { + callStack.remove(noCall); + Iterator i = callStack.iterator(); + assertEquals(call1, i.next()); + assertEquals(call2, i.next()); + assertEquals(call3, i.next()); + assertFalse(i.hasNext()); + } + + public void testContains() { + callStack.addLast(newCall); + assertTrue(callStack.contains(call1)); + assertTrue(callStack.contains(call2)); + assertTrue(callStack.contains(call3)); + assertTrue(callStack.contains(newCall)); + assertFalse(callStack.contains(noCall)); + } + + public void testPop() { + assertEquals(call1, callStack.pop()); + assertEquals(call2, callStack.pop()); + callStack.addNext(newCall); + + assertFalse(callStack.contains(call1)); + assertFalse(callStack.contains(call2)); + assertTrue(callStack.contains(call3)); + assertTrue(callStack.contains(newCall)); + + assertEquals(newCall, callStack.pop()); + assertTrue(callStack.contains(call3)); + assertFalse(callStack.contains(newCall)); + + assertEquals(call3, callStack.pop()); + assertFalse(callStack.contains(call3)); + + assertNull(callStack.pop()); + } + + public void testGetLastPopped() { + CallStack stakk = new CallStack(); + assertNull(stakk.getLastPopped()); + + Call call; + Call lastCall; + + call = callStack.pop(); + assertEquals(call1, call); + lastCall = callStack.getLastPopped(); + assertEquals(call1, lastCall); + assertEquals(call, lastCall); + + call = callStack.pop(); + assertEquals(call2, call); + lastCall = callStack.getLastPopped(); + assertEquals(call2, lastCall); + assertEquals(call, lastCall); + + call = callStack.pop(); + assertEquals(call3, call); + lastCall = callStack.getLastPopped(); + assertEquals(call3, lastCall); + assertEquals(call, lastCall); + + lastCall = callStack.getLastPopped(); + assertEquals(call3, lastCall); + assertEquals(call, lastCall); + + lastCall = callStack.getLastPopped(); + assertEquals(call3, lastCall); + assertEquals(call, lastCall); + + callStack.addLast(call1); + callStack.addLast(call2); + + lastCall = callStack.getLastPopped(); + assertEquals(call3, lastCall); + assertEquals(call, lastCall); + + call = callStack.pop(); + assertEquals(call1, call); + lastCall = callStack.getLastPopped(); + assertEquals(call1, lastCall); + assertEquals(call, lastCall); + + call = callStack.pop(); + assertEquals(call2, call); + lastCall = callStack.getLastPopped(); + assertEquals(call2, lastCall); + assertEquals(call, lastCall); + + lastCall = callStack.getLastPopped(); + assertEquals(call2, lastCall); + assertEquals(call, lastCall); + + lastCall = callStack.getLastPopped(); + assertEquals(call2, lastCall); + assertEquals(call, lastCall); + } + + private static class TestDocumentProcessor extends com.yahoo.docproc.SimpleDocumentProcessor { + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/CallbackTestCase.java b/docproc/src/test/java/com/yahoo/docproc/CallbackTestCase.java new file mode 100644 index 00000000000..c51682b77c8 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/CallbackTestCase.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class CallbackTestCase extends junit.framework.TestCase { + private DocumentPut put1; + private DocumentPut put2; + private List<DocumentOperation> operations = new ArrayList<>(2); + DocprocService service; + + + public void setUp() { + service = new DocprocService("callback"); + service.setCallStack(new CallStack().addNext(new TestCallbackDp())); + service.setInService(true); + + // Create documents + DocumentType type = new DocumentType("test"); + type.addField("status", DataType.STRING); + put1 = new DocumentPut(type, new DocumentId("doc:callback:test:1")); + put2 = new DocumentPut(type, new DocumentId("doc:callback:test:2")); + operations.add(new DocumentPut(type, new DocumentId("doc:callback:test:3"))); + operations.add(new DocumentPut(type, new DocumentId("doc:callback:test:4"))); + } + + public void testProcessingWithCallbackSingleDoc() { + ProcessingEndpoint drecv = new TestProcessingEndpoint(); + + service.process(put1, drecv); + while (service.doWork()) { } + + assertEquals(new StringFieldValue("received"), put1.getDocument().getFieldValue("status")); + } + + public void testProcessingWithCallbackMultipleDocs() { + ProcessingEndpoint drecv = new TestProcessingEndpoint(); + + service.process(toProcessing(operations), drecv); + while (service.doWork()) { } + + assertEquals(new StringFieldValue("received"), ((DocumentPut) operations.get(0)).getDocument().getFieldValue("status")); + assertEquals(new StringFieldValue("received"), ((DocumentPut) operations.get(1)).getDocument().getFieldValue("status")); + } + + private Processing toProcessing(List<DocumentOperation> documentOperations) { + Processing processing = new Processing(); + for (DocumentOperation op : documentOperations) + processing.addDocumentOperation(op); + return processing; + } + + public void testProcessingWithCallbackProcessing() { + ProcessingEndpoint drecv = new TestProcessingEndpoint(); + + Processing processing = new Processing("default", put2, service.getCallStack()); + + service.process(processing, drecv); + while (service.doWork()) { } + + assertEquals(new StringFieldValue("received"), put2.getDocument().getFieldValue("status")); + } + + public class TestProcessingEndpoint implements ProcessingEndpoint { + public void processingDone(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + ((DocumentPut)op).getDocument().setFieldValue("status", new StringFieldValue("received")); + } + } + + public void processingFailed(Processing processing, Exception exception) { + //do nothing here for now + } + } + + public static class TestCallbackDp extends com.yahoo.docproc.SimpleDocumentProcessor { + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/DocumentProcessingAbstractTestCase.java b/docproc/src/test/java/com/yahoo/docproc/DocumentProcessingAbstractTestCase.java new file mode 100644 index 00000000000..cce3460d3c4 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/DocumentProcessingAbstractTestCase.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * Convenience superclass of document processor test cases + * + * @author bratseth + */ +public abstract class DocumentProcessingAbstractTestCase extends junit.framework.TestCase { + + public DocumentProcessingAbstractTestCase(String name) { + super(name); + } + + /** + * Asserts that a document processing service works + */ + protected void assertProcessingWorks(DocprocService service) { + // Create documents + DocumentType type = new DocumentType("test"); + type.addField("test", DataType.STRING); + DocumentPut put1 = new DocumentPut(type, new DocumentId("doc:test:test:1")); + DocumentPut put2 = new DocumentPut(type, new DocumentId("doc:test:test:2")); + DocumentPut put3 = new DocumentPut(type, new DocumentId("doc:test:test:3")); + + // Process them + service.process(put1); + service.process(put2); + service.process(put3); + while (service.doWork()) {} + + // Verify + assertEquals(new StringFieldValue("done"), put1.getDocument().getFieldValue("test")); + assertEquals(new StringFieldValue("done"), put2.getDocument().getFieldValue("test")); + assertEquals(new StringFieldValue("done"), put3.getDocument().getFieldValue("test")); + } + + public static class TestDocumentProcessor1 extends DocumentProcessor { + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + if (processing.getVariable("processor1") == null) { + processing.setVariable("processor1", "called"); + return Progress.LATER; + } + processing.setVariable("processor1", "calledTwice"); + } + return Progress.DONE; + } + } + + public static class TestDocumentProcessor2 extends DocumentProcessor { + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + assertEquals("calledTwice", processing.getVariable("processor1")); + processing.setVariable("processor2", "called"); + } + return Progress.DONE; + } + } + + public static class TestDocumentProcessor3 extends DocumentProcessor { + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + assertEquals("called", processing.getVariable("processor2")); + if (processing.getVariable("processor3") == null) { + processing.setVariable("processor3", "called"); + return Progress.LATER; + } + ((DocumentPut)op).getDocument().setFieldValue("test", new StringFieldValue("done")); + } + return Progress.DONE; + } + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/EmptyProcessingTestCase.java b/docproc/src/test/java/com/yahoo/docproc/EmptyProcessingTestCase.java new file mode 100644 index 00000000000..060f3a9747c --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/EmptyProcessingTestCase.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import org.junit.Test; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class EmptyProcessingTestCase { + + @Test + public void emptyProcessing() { + DocprocService service = new DocprocService("juba"); + DocumentProcessor processor = new IncrementingDocumentProcessor(); + CallStack stack = new CallStack("juba"); + stack.addLast(processor); + service.setCallStack(stack); + service.setInService(true); + + Processing proc = new Processing(); + proc.setServiceName("juba"); + + service.process(proc); + + while (service.doWork()) { } + + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingTestCase.java b/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingTestCase.java new file mode 100644 index 00000000000..1b7a005350e --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * Tests a document processing where some processings fail with an exception + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class FailingDocumentProcessingTestCase extends junit.framework.TestCase { + + public FailingDocumentProcessingTestCase(String name) { + super(name); + } + + /** + * Tests chaining of some processors, and execution of the processors + * on some documents + */ + public void testFailingProcessing() { + // Set up service programmatically + DocprocService service = new DocprocService("failing"); + DocumentProcessor first = new SettingValueProcessor("done 1"); + DocumentProcessor second = new FailingProcessor("done 2"); + DocumentProcessor third = new SettingValueProcessor("done 3"); + service.setCallStack(new CallStack().addLast(first).addLast(second).addLast(third)); + service.setInService(true); + + assertProcessingWorks(service); + } + + protected void assertProcessingWorks(DocprocService service) { + // Create documents + DocumentType type = new DocumentType("test"); + type.addField("test", DataType.STRING); + DocumentPut put1 = new DocumentPut(type, new DocumentId("doc:failing:test:1")); + DocumentPut put2 = new DocumentPut(type, new DocumentId("doc:failing:test:2")); + DocumentPut put3 = new DocumentPut(type, new DocumentId("doc:failing:test:3")); + + // Process them + service.process(put1); + service.process(put2); + service.process(put3); + while (service.doWork()) {} + + // Verify + assertEquals(new StringFieldValue("done 3"), put1.getDocument().getFieldValue("test")); + assertEquals(new StringFieldValue("done 2"), put2.getDocument().getFieldValue("test")); // Due to exception in 2 + assertEquals(new StringFieldValue("done 3"), put3.getDocument().getFieldValue("test")); + } + + public static class SettingValueProcessor extends SimpleDocumentProcessor { + + private String value; + + public SettingValueProcessor(String value) { + this.value = value; + } + + @Override + public void process(DocumentPut put) { + put.getDocument().setFieldValue("test", value); + } + } + + public static class FailingProcessor extends SettingValueProcessor { + + public FailingProcessor(String name) { + super(name); + } + + @Override + public void process(DocumentPut put) { + super.process(put); + if (put.getId().toString().equals("doc:failing:test:2")) { + throw new HandledProcessingException("Failed at receiving document test:2"); + } + } + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingWithoutExceptionTestCase.java b/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingWithoutExceptionTestCase.java new file mode 100644 index 00000000000..1db47b0dfb2 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/FailingDocumentProcessingWithoutExceptionTestCase.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * Tests a document processing where some processings fail with an exception + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M. R. Rosenvinge</a> + */ +public class FailingDocumentProcessingWithoutExceptionTestCase extends junit.framework.TestCase { + + public FailingDocumentProcessingWithoutExceptionTestCase(String name) { + super(name); + } + + /** + * Tests chaining of some processors, and execution of the processors + * on some documents + */ + public void testFailingProcessing() { + // Set up service programmatically + DocprocService service = new DocprocService("failing-no-exception"); + DocumentProcessor first = new SettingValueProcessor("done 1"); + DocumentProcessor second = new FailingProcessor("done 2"); + DocumentProcessor third = new SettingValueProcessor("done 3"); + service.setCallStack(new CallStack().addLast(first).addLast(second).addLast(third)); + service.setInService(true); + + assertProcessingWorks(service); + } + + protected void assertProcessingWorks(DocprocService service) { + // Create documents + DocumentType type = new DocumentType("test"); + type.addField("test", DataType.STRING); + DocumentPut put1 = new DocumentPut(type, new DocumentId("doc:woexception:test:1")); + DocumentPut put2 = new DocumentPut(type, new DocumentId("doc:woexception:test:2")); + DocumentPut put3 = new DocumentPut(type, new DocumentId("doc:woexception:test:3")); + + // Process them + service.process(put1); + service.process(put2); + service.process(put3); + while (service.doWork()) {} + + // Verify + assertEquals(new StringFieldValue("done 3"), put1.getDocument().getFieldValue("test")); + assertEquals(new StringFieldValue("done 2"), put2.getDocument().getFieldValue("test")); // Due to PROCESSING_FAILED in 2 + assertEquals(new StringFieldValue("done 3"), put3.getDocument().getFieldValue("test")); + } + + public static class SettingValueProcessor extends DocumentProcessor { + + private String value; + + public SettingValueProcessor(String value) { + this.value = value; + } + + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + ((DocumentPut)op).getDocument().setFieldValue("test", value); + } + return Progress.DONE; + } + } + + public static class FailingProcessor extends SettingValueProcessor { + + public FailingProcessor(String name) { + super(name); + } + + @Override + public Progress process(Processing processing) { + super.process(processing); + for (DocumentOperation op : processing.getDocumentOperations()) { + if (op.getId().toString().equals("doc:woexception:test:2")) { + return DocumentProcessor.Progress.FAILED; + } + } + return DocumentProcessor.Progress.DONE; + } + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/FailingPermanentlyDocumentProcessingTestCase.java b/docproc/src/test/java/com/yahoo/docproc/FailingPermanentlyDocumentProcessingTestCase.java new file mode 100644 index 00000000000..0e84ba04647 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/FailingPermanentlyDocumentProcessingTestCase.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.datatypes.StringFieldValue; + +/** + * Tests a document processing where some processings fail permanently + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M. R. Rosenvinge</a> + */ +public class FailingPermanentlyDocumentProcessingTestCase extends junit.framework.TestCase { + + public FailingPermanentlyDocumentProcessingTestCase(String name) { + super(name); + } + + /** + * Tests chaining of some processors, and execution of the processors + * on some documents + */ + public void testFailingProcessing() { + // Set up service programmatically + DocprocService service = new DocprocService("failing-permanently"); + DocumentProcessor first = new SettingValueProcessor("done 1"); + DocumentProcessor second = new FailingProcessor("done 2"); + DocumentProcessor third = new SettingValueProcessor("done 3"); + service.setCallStack(new CallStack().addLast(first).addLast(second).addLast(third)); + service.setInService(true); + + assertProcessingWorks(service); + } + + protected void assertProcessingWorks(DocprocService service) { + // Create documents + DocumentType type = new DocumentType("test"); + type.addField("test", DataType.STRING); + DocumentPut put1 = new DocumentPut(type, new DocumentId("doc:permanentfailure:test:1")); + DocumentPut put2 = new DocumentPut(type, new DocumentId("doc:permanentfailure:test:2")); + DocumentPut put3 = new DocumentPut(type, new DocumentId("doc:permanentfailure:test:3")); + + // Process them + service.process(put1); + service.process(put2); + service.process(put3); + while (service.doWork()) {} + + // Verify + assertEquals(new StringFieldValue("done 3"), put1.getDocument().getFieldValue("test")); + assertEquals(new StringFieldValue("done 2"), put2.getDocument().getFieldValue("test")); // Due to PERMANENT_FAILURE in 2 + assertNull(put3.getDocument().getFieldValue("test")); //service is disabled now + assertFalse(service.isInService()); + + service.setInService(true); + while (service.doWork()) {} + + assertEquals(new StringFieldValue("done 3"), put3.getDocument().getFieldValue("test")); + assertTrue(service.isInService()); + } + + public static class SettingValueProcessor extends DocumentProcessor { + + private String value; + + public SettingValueProcessor(String value) { + this.value = value; + } + + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + ((DocumentPut)op).getDocument().setFieldValue("test", value); + } + return Progress.DONE; + } + } + + public static class FailingProcessor extends SettingValueProcessor { + + public FailingProcessor(String name) { + super(name); + } + + @Override + public Progress process(Processing processing) { + super.process(processing); + for (DocumentOperation op : processing.getDocumentOperations()) { + if (op.getId().toString().equals("doc:permanentfailure:test:2")) { + return DocumentProcessor.Progress.PERMANENT_FAILURE; + } + } + return DocumentProcessor.Progress.DONE; + } + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/FailingWithErrorTestCase.java b/docproc/src/test/java/com/yahoo/docproc/FailingWithErrorTestCase.java new file mode 100644 index 00000000000..184b0aa9e88 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/FailingWithErrorTestCase.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class FailingWithErrorTestCase extends junit.framework.TestCase { + + public void testErrors() { + DocprocService service = new DocprocService("failing"); + DocumentProcessor first = new ErrorThrowingProcessor(); + service.setCallStack(new CallStack().addLast(first)); + service.setInService(true); + + DocumentType type = new DocumentType("test"); + type.addField("test", DataType.STRING); + DocumentPut put = new DocumentPut(type, new DocumentId("doc:failing:test:1")); + put.getDocument().setFieldValue("test", "foobar"); + + service.process(put); + assertEquals(1, service.getQueueSize()); + try { + while (service.doWork()) { } + fail("Should have gotten OOME here"); + } catch (Throwable t) { + //we don't want a finally block in doWork()! + assertEquals(0, service.getQueueSize()); + } + assertEquals(0, service.getQueueSize()); + + } + + private class ErrorThrowingProcessor extends DocumentProcessor { + @Override + public Progress process(Processing processing) { + throw new OutOfMemoryError("Einar is out of mem"); + } + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/IncrementingDocumentProcessor.java b/docproc/src/test/java/com/yahoo/docproc/IncrementingDocumentProcessor.java new file mode 100644 index 00000000000..660daec67d8 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/IncrementingDocumentProcessor.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DocumentPut; + +/** + * Document processor used for a simple, silly test. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IncrementingDocumentProcessor extends SimpleDocumentProcessor { + int counter = 0; + + @Override + public void process(DocumentPut put) { + System.err.println(counter + " DocumentPut: " + put); + counter++; + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/NotAcceptingNewProcessingsTestCase.java b/docproc/src/test/java/com/yahoo/docproc/NotAcceptingNewProcessingsTestCase.java new file mode 100644 index 00000000000..9e5f7a43c74 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/NotAcceptingNewProcessingsTestCase.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class NotAcceptingNewProcessingsTestCase extends junit.framework.TestCase { + + public void testNotAccepting() { + DocprocService service = new DocprocService("habla"); + service.setCallStack(new CallStack()); + service.setInService(true); + + service.process(new Processing()); + assertEquals(1, service.getQueueSize()); + + service.setAcceptingNewProcessings(false); + + try { + service.process(new Processing()); + fail("Should have gotten IllegalStateException here"); + } catch (IllegalStateException ise) { + //ok! + } + assertEquals(1, service.getQueueSize()); + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/ProcessingTestCase.java b/docproc/src/test/java/com/yahoo/docproc/ProcessingTestCase.java new file mode 100644 index 00000000000..296c8d163e5 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/ProcessingTestCase.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DocumentOperation; +import org.junit.Test; + +import java.util.Iterator; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.10 + */ +public class ProcessingTestCase { + + @Test + public void serviceName() { + assertThat(new Processing().getServiceName(), is("default")); + assertThat(new Processing("foobar", (DocumentOperation) null, null).getServiceName(), is("foobar")); + } + + @Test + public void contextVariables() { + Processing p = new Processing(); + + p.setVariable("foo", "banana"); + p.setVariable("bar", "apple"); + + Iterator<Map.Entry<String, Object>> it = p.getVariableAndNameIterator(); + assertThat(it.hasNext(), is(true)); + assertThat(it.next(), not(nullValue())); + assertThat(it.hasNext(), is(true)); + assertThat(it.next(), not(nullValue())); + assertThat(it.hasNext(), is(false)); + + assertThat(p.hasVariable("foo"), is(true)); + assertThat(p.hasVariable("bar"), is(true)); + assertThat(p.hasVariable("baz"), is(false)); + + assertThat(p.removeVariable("foo"), is("banana")); + + p.clearVariables(); + + it = p.getVariableAndNameIterator(); + assertThat(it.hasNext(), is(false)); + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java new file mode 100644 index 00000000000..d8398f1fe47 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/ProcessingUpdateTestCase.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.update.AssignValueUpdate; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.document.update.ValueUpdate; + +import java.util.List; +import java.util.StringTokenizer; + +/** + * Simple test case for testing that processing of both documents and + * document updates works. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class ProcessingUpdateTestCase extends junit.framework.TestCase { + + private DocumentPut put; + private DocumentUpdate update; + + private DocumentTypeManager dtm; + + public void testProcessingUpdates() { + DocumentType articleType = new DocumentType("article"); + articleType.addField(new Field("body", DataType.STRING, true)); + articleType.addField(new Field("title", DataType.STRING, true)); + dtm = new DocumentTypeManager(); + dtm.registerDocumentType(articleType); + + put = new DocumentPut(articleType, "doc:banana:apple"); + put.getDocument().setFieldValue("body", "this is the body of the article, blah blah blah"); + FieldUpdate upd = FieldUpdate.createAssign(articleType.getField("body"), new StringFieldValue("this is the updated body of the article, blahdi blahdi blahdi")); + update = new DocumentUpdate(articleType, new DocumentId("doc:grape:orange")); + update.addFieldUpdate(upd); + + DocprocService service = new DocprocService("update"); + DocumentProcessor firstP = new TitleDocumentProcessor(); + service.setCallStack(new CallStack().addLast(firstP)); + service.setInService(true); + + + + Processing p = new Processing(); + p.addDocumentOperation(put); + p.addDocumentOperation(update); + + service.process(p); + + while (service.doWork()) { } + + List<DocumentOperation> operations = p.getDocumentOperations(); + Document first = ((DocumentPut)operations.get(0)).getDocument(); + assertEquals(new StringFieldValue("this is the body of the article, blah blah blah"), first.getFieldValue("body")); + assertEquals(new StringFieldValue("body blah blah blah "), first.getFieldValue("title")); + + DocumentUpdate second = (DocumentUpdate) operations.get(1); + FieldUpdate firstUpd = second.getFieldUpdate(0); + assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, firstUpd.getValueUpdate(0).getValueUpdateClassID()); + assertEquals(new StringFieldValue("this is the updated body of the article, blahdi blahdi blahdi"), firstUpd.getValueUpdate(0) + .getValue()); + + FieldUpdate secondUpd = second.getFieldUpdate(1); + assertEquals(ValueUpdate.ValueUpdateClassID.ASSIGN, secondUpd.getValueUpdate(0).getValueUpdateClassID()); + assertEquals(new StringFieldValue("body blahdi blahdi blahdi "), secondUpd.getValueUpdate(0).getValue()); + } + + private class TitleDocumentProcessor extends SimpleDocumentProcessor { + @Override + public void process(DocumentPut doc) { + put.getDocument().setFieldValue("title", extractTitle(put.getDocument().getFieldValue("body").toString())); + } + + @Override + public void process(DocumentUpdate upd) { + FieldUpdate bodyFieldUpdate = upd.getFieldUpdate("body"); + AssignValueUpdate au = (AssignValueUpdate) bodyFieldUpdate.getValueUpdate(0); + FieldUpdate titleUpd = FieldUpdate.createAssign(upd.getType().getField("title"), new StringFieldValue(extractTitle(((StringFieldValue) au.getValue()).getString()))); + upd.addFieldUpdate(titleUpd); + } + + private String extractTitle(String body) { + if (body == null) return null; + StringTokenizer strTok = new StringTokenizer(body, " "); + String title = ""; + while (strTok.hasMoreTokens()) { + String word = strTok.nextToken(); + if (word.startsWith("b")) title += word + " "; + } + return title; + } + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessingTestCase.java b/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessingTestCase.java new file mode 100644 index 00000000000..8ca553b9290 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessingTestCase.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.docproc.Accesses.Field.Tree; + +/** + * Tests the basic operation of the docproc service + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class SimpleDocumentProcessingTestCase extends DocumentProcessingAbstractTestCase { + + public SimpleDocumentProcessingTestCase(String name) { + super(name); + } + + /** + * Tests chaining of some processors, and execution of the processors + * on some documents + */ + public void testSimpleProcessing() { + // Set up service programmatically + DocprocService service = new DocprocService("simple"); + DocumentProcessor first = new TestDocumentProcessor1(); + DocumentProcessor second = new TestDocumentProcessor2(); + DocumentProcessor third = new TestDocumentProcessor3(); + service.setCallStack(new CallStack().addLast(first).addLast(second).addLast(third)); + service.setInService(true); + + assertProcessingWorks(service); + } + + public void testAnnotationBasic() { + Accesses accesses = MyDocProc.class.getAnnotation(Accesses.class); + After after = MyDocProc.class.getAnnotation(After.class); + assertNotNull(accesses); + assertNotNull(after); + assertEquals(after.value()[0], "MyOtherDocProc"); + assertEquals(after.value()[1], "AnotherDocProc"); + assertEquals(accesses.value()[0].name(), "myField1"); + assertEquals(accesses.value()[1].annotations()[0].consumes()[0], "word"); + assertEquals(accesses.value()[1].annotations()[0].consumes()[1], "phrase"); + } + + @Accesses({ @Accesses.Field(name = "myField1", dataType = "string", + description = "What is done on field myField1", + annotations = @Tree(produces = { "word", "sentence" }, consumes = "word")), + @Accesses.Field(name = "myField2", dataType = "string", + description = "What is done on field myField2", + annotations = @Tree(consumes = { "word", "phrase" })) }) + @After({ "MyOtherDocProc", "AnotherDocProc" }) + public static class MyDocProc extends DocumentProcessor { + + @Override + public Progress process(Processing processing) { + // TODO Auto-generated method stub + return null; + } + } +} + diff --git a/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessorTestCase.java b/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessorTestCase.java new file mode 100644 index 00000000000..aae98dd22e7 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/SimpleDocumentProcessorTestCase.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.container.StatisticsConfig; +import com.yahoo.docproc.jdisc.metric.NullMetric; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentRemove; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.idstring.UserDocIdString; +import com.yahoo.statistics.StatisticsImpl; +import org.junit.Test; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNull.nullValue; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SimpleDocumentProcessorTestCase { + + private static DocprocService setupDocprocService(SimpleDocumentProcessor processor) { + CallStack stack = new CallStack("default", new StatisticsImpl(new StatisticsConfig(new StatisticsConfig.Builder())), new NullMetric()); + stack.addLast(processor); + DocprocService service = new DocprocService("default"); + service.setCallStack(stack); + service.setInService(true); + return service; + } + + private static Processing getProcessing(DocumentOperation... operations) { + Processing processing = new Processing(); + + for (DocumentOperation op : operations) { + processing.addDocumentOperation(op); + } + + return processing; + } + + @Test + public void requireThatProcessingMultipleOperationsWork() { + DocumentType type = new DocumentType("foobar"); + type.addField("title", DataType.STRING); + + Processing p = getProcessing(new DocumentPut(type, "doc:this:is:a:document"), + new DocumentUpdate(type, "doc:this:is:an:update"), + new DocumentRemove(new DocumentId("doc:this:is:a:remove"))); + + DocprocService service = setupDocprocService(new VerySimpleDocumentProcessor()); + service.getExecutor().process(p); + + assertThat(p.getDocumentOperations().size(), is(3)); + assertThat(p.getDocumentOperations().get(0) instanceof DocumentPut, is(true)); + assertThat(((DocumentPut) p.getDocumentOperations().get(0)).getDocument().getFieldValue("title").getWrappedValue(), + is("processed")); + assertThat(p.getDocumentOperations().get(1) instanceof DocumentUpdate, is(true)); + assertThat(p.getDocumentOperations().get(2) instanceof DocumentRemove, is(true)); + assertThat(p.getDocumentOperations().get(2).getId().toString(), + is("userdoc:foobar:1234:something")); + } + + @Test + public void requireThatProcessingSingleOperationWorks() { + DocumentType type = new DocumentType("foobar"); + type.addField("title", DataType.STRING); + + Processing p = getProcessing(new DocumentPut(type, "doc:this:is:a:document")); + DocprocService service = setupDocprocService(new VerySimpleDocumentProcessor()); + service.getExecutor().process(p); + + assertThat(p.getDocumentOperations().size(), is(1)); + assertThat(p.getDocumentOperations().get(0) instanceof DocumentPut, is(true)); + assertThat(((DocumentPut) p.getDocumentOperations().get(0)).getDocument().getFieldValue("title").getWrappedValue(), + is("processed")); + } + + @Test + public void requireThatThrowingTerminatesIteration() { + DocumentType type = new DocumentType("foobar"); + type.addField("title", DataType.STRING); + + Processing p = getProcessing(new DocumentPut(type, "doc:this:is:a:document"), + new DocumentRemove(new DocumentId("doc:this:is:a:remove")), + new DocumentPut(type, "doc:this:is:a:document2")); + + DocprocService service = setupDocprocService(new SimpleDocumentProcessorThrowingOnRemovesAndUpdates()); + try { + service.getExecutor().process(p); + } catch (RuntimeException re) { + //ok + } + + assertThat(p.getDocumentOperations().size(), is(3)); + assertThat(p.getDocumentOperations().get(0) instanceof DocumentPut, is(true)); + assertThat(((DocumentPut) p.getDocumentOperations().get(0)).getDocument().getFieldValue("title").getWrappedValue(), + is("processed")); + assertThat(p.getDocumentOperations().get(1) instanceof DocumentRemove, is(true)); + assertThat(p.getDocumentOperations().get(1).getId().toString(), + is("doc:this:is:a:remove")); + assertThat(p.getDocumentOperations().get(2) instanceof DocumentPut, is(true)); + assertThat(((DocumentPut) p.getDocumentOperations().get(2)).getDocument().getFieldValue("title"), + nullValue()); + + + } + + public static class VerySimpleDocumentProcessor extends SimpleDocumentProcessor { + + @Override + public void process(DocumentPut put) { + put.getDocument().setFieldValue("title", new StringFieldValue("processed")); + } + + @Override + public void process(DocumentRemove remove) { + remove.getId().setId(new UserDocIdString("foobar", 1234L, "something")); + } + + @Override + public void process(DocumentUpdate update) { + update.clearFieldUpdates(); + } + + } + + public static class SimpleDocumentProcessorThrowingOnRemovesAndUpdates extends SimpleDocumentProcessor { + + @Override + public void process(DocumentPut put) { + put.getDocument().setFieldValue("title", new StringFieldValue("processed")); + } + + @Override + public void process(DocumentRemove remove) { + throw new RuntimeException("oh no."); + } + + @Override + public void process(DocumentUpdate update) { + throw new RuntimeException("oh no."); + } + + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/TransientFailureTestCase.java b/docproc/src/test/java/com/yahoo/docproc/TransientFailureTestCase.java new file mode 100644 index 00000000000..076a8610c4a --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/TransientFailureTestCase.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentOperation; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class TransientFailureTestCase extends junit.framework.TestCase { + DocumentType type; + + public void setUp() { + type = new DocumentType("test"); + type.addField("boo", DataType.STRING); + } + + public void testTransientFailures() { + DocprocService service = new DocprocService("transfail"); + CallStack stack = new CallStack(); + stack.addNext(new OkDocProc()).addNext(new TransientFailDocProc()); + service.setCallStack(stack); + service.setInService(true); + + EndpointSupportingTransientFailures endpoint = new EndpointSupportingTransientFailures(); + + DocumentPut put; + + put = new DocumentPut(type, new DocumentId("doc:transfail:bad")); + service.process(put, endpoint); + while (service.doWork()) { } + assertEquals(0, endpoint.numOk); + assertEquals(1, endpoint.numTransientFail); + assertEquals(0, endpoint.numFail); + + put = new DocumentPut(type, new DocumentId("doc:transfail:verybad")); + service.process(put, endpoint); + while (service.doWork()) { } + assertEquals(0, endpoint.numOk); + assertEquals(1, endpoint.numTransientFail); + assertEquals(1, endpoint.numFail); + + put = new DocumentPut(type, new DocumentId("doc:transfail:good")); + service.process(put, endpoint); + while (service.doWork()) { } + assertEquals(1, endpoint.numOk); + assertEquals(1, endpoint.numTransientFail); + assertEquals(1, endpoint.numFail); + + put = new DocumentPut(type, new DocumentId("doc:transfail:veryverybad")); + service.process(put, endpoint); + while (service.doWork()) { } + assertEquals(1, endpoint.numOk); + assertEquals(1, endpoint.numTransientFail); + assertEquals(2, endpoint.numFail); + } + + private static class OkDocProc extends SimpleDocumentProcessor { + } + + private static class TransientFailDocProc extends DocumentProcessor { + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + if (op.getId().toString().equals("doc:transfail:bad")) { + throw new TransientFailureException("sorry, try again later"); + } else if (op.getId().toString().equals("doc:transfail:verybad")) { + return Progress.FAILED; + } else if (op.getId().toString().equals("doc:transfail:veryverybad")) { + return Progress.PERMANENT_FAILURE; + } + } + return Progress.DONE; + } + } + + private static class EndpointSupportingTransientFailures implements ProcessingEndpoint { + private volatile int numOk = 0; + private volatile int numTransientFail = 0; + private volatile int numFail = 0; + + public void processingDone(Processing processing) { + numOk++; + } + + public void processingFailed(Processing processing, Exception exception) { + if (exception instanceof TransientFailureException) { + numTransientFail++; + } else { + numFail++; + } + } + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutorTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutorTestCase.java new file mode 100644 index 00000000000..f928ec4febd --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocprocThreadPoolExecutorTestCase.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.jdisc; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocprocThreadPoolExecutorTestCase { + private final Set<Long> threadIds = Collections.synchronizedSet(new HashSet<Long>()); + + @Test + public void threadPool() throws InterruptedException { + int numThreads = 8; + int numTasks = 200; + + LinkedBlockingQueue<Runnable> q = new LinkedBlockingQueue<>(); + DocprocThreadManager mgr = new DocprocThreadManager(1000l); + DocprocThreadPoolExecutor pool = new DocprocThreadPoolExecutor(numThreads, q, mgr); + + List<MockedDocumentProcessingTask> tasks = new ArrayList<>(numTasks); + for (int i = 0; i < numTasks; i++) { + tasks.add(new MockedDocumentProcessingTask()); + } + + for (int i = 0; i < numTasks; i++) { + pool.execute(tasks.get(i)); + } + pool.shutdown(); + pool.awaitTermination(120L, TimeUnit.SECONDS); + + for (int i = 0; i < numTasks; i++) { + assertTrue(tasks.get(i).hasBeenRun()); + } + + System.err.println(threadIds); + assertEquals(numThreads, threadIds.size()); + } + + private class MockedDocumentProcessingTask extends DocumentProcessingTask { + private boolean hasBeenRun = false; + + public MockedDocumentProcessingTask() { + super(null, null, null); + } + + @Override + public void run() { + threadIds.add(Thread.currentThread().getId()); + System.err.println(System.currentTimeMillis() + " MOCK Thread " + Thread.currentThread().getId() + " running task " + this); + for (int i = 0; i < 100000; i++) { + Math.sin((double) (System.currentTimeMillis() / 10000L)); + } + System.err.println(System.currentTimeMillis() + " MOCK Thread " + Thread.currentThread().getId() + " DONE task " + this); + hasBeenRun = true; + } + + @Override + public int getApproxSize() { + return 333; + } + + @Override + public String toString() { + return "seqNum " + getSeqNum(); + } + + public boolean hasBeenRun() { + return hasBeenRun; + } + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java new file mode 100644 index 00000000000..271ad09ab1d --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerAllMessageTypesTestCase.java @@ -0,0 +1,230 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.jdisc; + +import com.yahoo.collections.Pair; +import com.yahoo.docproc.CallStack; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.*; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.documentapi.messagebus.protocol.BatchDocumentUpdateMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.vdslib.DocumentList; +import com.yahoo.vdslib.Entry; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentProcessingHandlerAllMessageTypesTestCase extends DocumentProcessingHandlerTestBase { + + private static final String FOOBAR = "foobar"; + private final DocumentType type; + + public DocumentProcessingHandlerAllMessageTypesTestCase() { + this.type = new DocumentType("baz"); + this.type.addField(new Field("blahblah", DataType.STRING)); + this.type.addField(new Field("defaultWait", DataType.INT)); + this.type.addField(new Field("customWait", DataType.INT)); + } + + @Test + public void testMessages() throws InterruptedException { + get(); + put(); + remove(); + update(); + batchDocumentUpdate(); + } + + private void get() throws InterruptedException { + GetDocumentMessage message = new GetDocumentMessage(new DocumentId("doc:this:is:a:test"), "fieldset?"); + + assertTrue(sendMessage(FOOBAR, message)); + + Message result = remoteServer.awaitMessage(60, TimeUnit.SECONDS); + assertNotNull(result); + remoteServer.ackMessage(result); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + + assertThat(result, instanceOf(GetDocumentMessage.class)); + + assertFalse(reply.hasErrors()); + } + + + private void put() throws InterruptedException { + Document document = new Document(getType(), "doc:baz:foo"); + document.setFieldValue("blahblah", new StringFieldValue("This is a test.")); + PutDocumentMessage message = new PutDocumentMessage(new DocumentPut(document)); + + assertTrue(sendMessage(FOOBAR, message)); + + Message result = remoteServer.awaitMessage(60, TimeUnit.SECONDS); + assertNotNull(result); + remoteServer.ackMessage(result); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + + assertThat(result, instanceOf(PutDocumentMessage.class)); + PutDocumentMessage outputMsg = (PutDocumentMessage) result; + assertThat(((StringFieldValue) outputMsg.getDocumentPut().getDocument().getFieldValue("blahblah")).getString(), is("THIS IS A TEST.")); + + assertFalse(reply.hasErrors()); + } + + private void remove() throws InterruptedException { + RemoveDocumentMessage message = new RemoveDocumentMessage(new DocumentId("doc:12345:6789")); + + assertTrue(sendMessage(FOOBAR, message)); + + Message result = remoteServer.awaitMessage(60, TimeUnit.SECONDS); + assertNotNull(result); + remoteServer.ackMessage(result); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + + assertThat(result, instanceOf(RemoveDocumentMessage.class)); + RemoveDocumentMessage outputMsg = (RemoveDocumentMessage) result; + assertThat(outputMsg.getDocumentId().toString(), is("doc:12345:6789")); + + assertFalse(reply.hasErrors()); + } + + private void update() throws InterruptedException { + DocumentUpdate documentUpdate = new DocumentUpdate(getType(), "doc:baz:foo"); + UpdateDocumentMessage message = new UpdateDocumentMessage(documentUpdate); + + assertTrue(sendMessage(FOOBAR, message)); + + Message result = remoteServer.awaitMessage(60, TimeUnit.SECONDS); + assertNotNull(result); + remoteServer.ackMessage(result); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + + assertThat(result, instanceOf(UpdateDocumentMessage.class)); + UpdateDocumentMessage outputMsg = (UpdateDocumentMessage) result; + assertThat(outputMsg.getDocumentUpdate().getId().toString(), is("doc:baz:foo")); + + assertFalse(reply.hasErrors()); + } + + private void batchDocumentUpdate() throws InterruptedException { + DocumentUpdate doc1 = new DocumentUpdate(getType(), new DocumentId("userdoc:test:12345:multi:1")); + DocumentUpdate doc2 = new DocumentUpdate(getType(), new DocumentId("userdoc:test:12345:multi:2")); + + Field testField = getType().getField("blahblah"); + doc1.addFieldUpdate(FieldUpdate.createAssign(testField, new StringFieldValue("1 not yet processed"))); + doc2.addFieldUpdate(FieldUpdate.createAssign(testField, new StringFieldValue("2 not yet processed"))); + + BatchDocumentUpdateMessage message = new BatchDocumentUpdateMessage(12345); + message.addUpdate(doc1); + message.addUpdate(doc2); + + assertTrue(sendMessage(FOOBAR, message)); + + Message remote1 = remoteServer.awaitMessage(60, TimeUnit.SECONDS); + assertTrue(remote1 instanceof UpdateDocumentMessage); + remoteServer.ackMessage(remote1); + assertNull(driver.client().awaitReply(100, TimeUnit.MILLISECONDS)); + + Message remote2 = remoteServer.awaitMessage(60, TimeUnit.SECONDS); + assertTrue(remote2 instanceof UpdateDocumentMessage); + remoteServer.ackMessage(remote2); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + assertFalse(reply.hasErrors()); + } + + @Override + public List<Pair<String,CallStack>> getCallStacks() { + CallStack stack = new CallStack(); + stack.addLast(new YallaDocumentProcessor()); + stack.addLast(new WaitingDefaultDocumentProcessor()); + stack.addLast(new WaitingCustomDocumentProcessor()); + + ArrayList<Pair<String, CallStack>> stacks = new ArrayList<>(1); + stacks.add(new Pair<>(FOOBAR, stack)); + return stacks; + } + + @Override + public DocumentType getType() { + return type; + } + + private static class YallaDocumentProcessor extends DocumentProcessor { + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + if (op instanceof DocumentPut) { + Document doc = ((DocumentPut) op).getDocument(); + for (Field f : doc.getDataType().fieldSet()) { + FieldValue val = doc.getFieldValue(f); + if (val instanceof StringFieldValue) { + StringFieldValue sf = (StringFieldValue) val; + doc.setFieldValue(f, new StringFieldValue(sf.getString().toUpperCase())); + } + } + } + //don't touch updates or removes + } + return Progress.DONE; + } + } + + private static abstract class WaitingDocumentProcessor extends DocumentProcessor { + protected Progress laterProgress; + protected String waitKey; + + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + if (op instanceof DocumentPut) { + Document doc = ((DocumentPut) op).getDocument(); + if (doc.getFieldValue(waitKey) == null) { + System.out.println(this.getClass().getSimpleName() + ": returning LATER for " + doc); + doc.setFieldValue(waitKey, new IntegerFieldValue(1)); + return laterProgress; + } + } + //don't touch updates or removes + } + return Progress.DONE; + } + } + + private static class WaitingDefaultDocumentProcessor extends WaitingDocumentProcessor { + private WaitingDefaultDocumentProcessor() { + laterProgress = Progress.LATER; + waitKey = "defaultWait"; + } + } + + private static class WaitingCustomDocumentProcessor extends WaitingDocumentProcessor { + private WaitingCustomDocumentProcessor() { + laterProgress = Progress.later(60); + waitKey = "customWait"; + } + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerBasicTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerBasicTestCase.java new file mode 100644 index 00000000000..d308d0a484a --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerBasicTestCase.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.jdisc; + +import com.yahoo.collections.Pair; +import com.yahoo.docproc.CallStack; +import com.yahoo.docproc.SimpleDocumentProcessor; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.Field; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentProcessingHandlerBasicTestCase extends DocumentProcessingHandlerTestBase { + private DocumentType type; + + public DocumentProcessingHandlerBasicTestCase() { + this.type = new DocumentType("yalla"); + this.type.addField(new Field("blahblah", DataType.STRING)); + } + + @Test + public void testPut() throws InterruptedException { + Document document = new Document(getType(), "doc:yalla:balla"); + document.setFieldValue("blahblah", new StringFieldValue("This is a test.")); + PutDocumentMessage message = new PutDocumentMessage(new DocumentPut(document)); + + assertTrue(sendMessage("foobar", message)); + + Message msg = remoteServer.awaitMessage(60, TimeUnit.SECONDS); + assertNotNull(msg); + remoteServer.ackMessage(msg); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + + assertThat((msg instanceof PutDocumentMessage), is(true)); + PutDocumentMessage put = (PutDocumentMessage) msg; + Document outDoc = put.getDocumentPut().getDocument(); + + assertThat(document, equalTo(outDoc)); + assertFalse(reply.hasErrors()); + } + + @Override + public List<Pair<String,CallStack>> getCallStacks() { + CallStack stack = new CallStack(); + stack.addLast(new TestDocumentProcessor()); + + ArrayList<Pair<String, CallStack>> stacks = new ArrayList<>(1); + stacks.add(new Pair<>("foobar", stack)); + return stacks; + } + + @Override + public DocumentType getType() { + return type; + } + + public static class TestDocumentProcessor extends SimpleDocumentProcessor { + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerForkTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerForkTestCase.java new file mode 100644 index 00000000000..b637f35af4f --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerForkTestCase.java @@ -0,0 +1,215 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.jdisc; + +import com.yahoo.collections.Pair; +import com.yahoo.docproc.CallStack; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.*; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.WriteDocumentReply; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.vdslib.DocumentList; +import com.yahoo.vdslib.Entry; +import org.junit.Test; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentProcessingHandlerForkTestCase extends DocumentProcessingHandlerTestBase { + + private static final String TOMANYALLINSAMEBUCKET = "tomanyallinsamebucket"; + private static final String TOMANYSOMEINSAMEBUCKET = "tomanysomeinsamebucket"; + private static final String TOMANY = "many"; + private static final String TOONE = "toone"; + private static final String TOZERO = "tozero"; + private final DocumentType type; + + public DocumentProcessingHandlerForkTestCase() { + this.type = new DocumentType("baz"); + this.type.addField(new Field("blahblah", DataType.STRING)); + } + + @Override + public DocumentType getType() { + return type; + } + + @Test + public void testMessages() throws InterruptedException { + putToFourPuts(); + putToManyAllInSameBucket(); + putToManySomeInSameBucket(); + putToOne(); + putToZero(); + } + + private void putToManyAllInSameBucket() throws InterruptedException { + assertPutMessages(createPutDocumentMessage(), TOMANYALLINSAMEBUCKET, + "userdoc:123456:11111:foo:er:bra", + "userdoc:123456:11111:foo:trallala", + "userdoc:123456:11111:foo:a"); + } + + private void putToManySomeInSameBucket() throws InterruptedException { + assertPutMessages(createPutDocumentMessage(), TOMANYSOMEINSAMEBUCKET, + "userdoc:123456:7890:bar:er:bra", + "doc:foo:bar:er:ja", + "userdoc:567890:1234:a", + "doc:foo:bar:hahahhaa", + "userdoc:123456:7890:a:a", + "doc:foo:bar:aa", + "userdoc:567890:1234:bar:ala", + "doc:foo:bar:sdfgsaa", + "userdoc:123456:7890:bar:tralsfa", + "doc:foo:bar:dfshaa"); + } + + private void putToFourPuts() throws InterruptedException { + assertPutMessages(createPutDocumentMessage(), TOMANY, + "doc:foo:bar:er:bra", + "doc:foo:bar:er:ja", + "doc:foo:bar:hahahhaa", + "doc:foo:bar:trallala"); + } + + private void putToOne() throws InterruptedException { + assertPutMessages(createPutDocumentMessage(), TOONE, + "doc:baz:bar"); + } + + private void putToZero() throws InterruptedException { + assertTrue(sendMessage(TOZERO, createPutDocumentMessage())); + + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertTrue(reply instanceof WriteDocumentReply); + assertFalse(reply.hasErrors()); + } + + @Override + protected List<Pair<String, CallStack>> getCallStacks() { + ArrayList<Pair<String, CallStack>> stacks = new ArrayList<>(5); + stacks.add(new Pair<>(TOMANYALLINSAMEBUCKET, new CallStack().addLast(new OneToManyDocumentsAllInSameBucketProcessor()))); + stacks.add(new Pair<>(TOMANYSOMEINSAMEBUCKET, new CallStack().addLast(new OneToManyDocumentsSomeInSameBucketProcessor()))); + stacks.add(new Pair<>(TOMANY, new CallStack().addLast(new OneToManyDocumentsProcessor()))); + stacks.add(new Pair<>(TOONE, new CallStack().addLast(new OneToOneDocumentsProcessor()))); + stacks.add(new Pair<>(TOZERO, new CallStack().addLast(new OneToZeroDocumentsProcessor()))); + return stacks; + } + + protected PutDocumentMessage createPutDocumentMessage() { + Document document = new Document(getType(), "doc:baz:bar"); + document.setFieldValue("blahblah", new StringFieldValue("This is a test.")); + return new PutDocumentMessage(new DocumentPut(document)); + } + + private void assertPutMessages(DocumentMessage msg, String route, String... expected) throws InterruptedException { + msg.getTrace().setLevel(9); + assertTrue(sendMessage(route, msg)); + + String[] actual = new String[expected.length]; + for (int i = 0; i < expected.length; ++i) { + Message remoteMsg = remoteServer.awaitMessage(60, TimeUnit.SECONDS); + assertTrue(remoteMsg instanceof PutDocumentMessage); + remoteMsg.getTrace().trace(1, "remoteServer.ack(" + expected[i] + ")"); + remoteServer.ackMessage(remoteMsg); + actual[i] = ((PutDocumentMessage)remoteMsg).getDocumentPut().getDocument().getId().toString(); + } + assertNull(remoteServer.awaitMessage(100, TimeUnit.MILLISECONDS)); + + Arrays.sort(expected); + Arrays.sort(actual); + assertArrayEquals(expected, actual); + + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + assertFalse(reply.hasErrors()); + String trace = reply.getTrace().toString(); + for (String documentId : expected) { + assertTrue("missing trace for document '" + documentId + "'\n" + trace, + trace.contains("remoteServer.ack(" + documentId + ")")); + } + if (expected.length == 1) { + assertFalse("unexpected fork in trace for single document\n" + trace, + trace.contains("<fork>")); + } else { + assertTrue("missing fork in trace for " + expected.length + " split\n" + trace, + trace.contains("<fork>")); + } + } + + public class OneToOneDocumentsProcessor extends DocumentProcessor { + + @Override + public Progress process(Processing processing) { + return Progress.DONE; + } + } + + public class OneToManyDocumentsProcessor extends DocumentProcessor { + + @Override + public Progress process(Processing processing) { + List<DocumentOperation> operations = processing.getDocumentOperations(); + operations.clear(); + operations.add(new DocumentPut(type, "doc:foo:bar:er:bra")); + operations.add(new DocumentPut(type, "doc:foo:bar:er:ja")); + operations.add(new DocumentPut(type, "doc:foo:bar:trallala")); + operations.add(new DocumentPut(type, "doc:foo:bar:hahahhaa")); + return Progress.DONE; + } + } + + public class OneToZeroDocumentsProcessor extends DocumentProcessor { + + @Override + public Progress process(Processing processing) { + processing.getDocumentOperations().clear(); + return Progress.DONE; + } + } + + public class OneToManyDocumentsSomeInSameBucketProcessor extends DocumentProcessor { + + @Override + public Progress process(Processing processing) { + List<DocumentOperation> operations = processing.getDocumentOperations(); + operations.clear(); + operations.add(new DocumentPut(type, "userdoc:123456:7890:bar:er:bra")); + operations.add(new DocumentPut(type, "doc:foo:bar:er:ja")); + operations.add(new DocumentPut(type, "userdoc:567890:1234:a")); + operations.add(new DocumentPut(type, "doc:foo:bar:hahahhaa")); + operations.add(new DocumentPut(type, "userdoc:123456:7890:a:a")); + operations.add(new DocumentPut(type, "doc:foo:bar:aa")); + operations.add(new DocumentPut(type, "userdoc:567890:1234:bar:ala")); + operations.add(new DocumentPut(type, "doc:foo:bar:sdfgsaa")); + operations.add(new DocumentPut(type, "userdoc:123456:7890:bar:tralsfa")); + operations.add(new DocumentPut(type, "doc:foo:bar:dfshaa")); + return Progress.DONE; + } + + } + + public class OneToManyDocumentsAllInSameBucketProcessor extends DocumentProcessor { + + @Override + public Progress process(Processing processing) { + List<DocumentOperation> docs = processing.getDocumentOperations(); + docs.clear(); + docs.add(new DocumentPut(type, "userdoc:123456:11111:foo:er:bra")); + docs.add(new DocumentPut(type, "userdoc:123456:11111:foo:trallala")); + docs.add(new DocumentPut(type, "userdoc:123456:11111:foo:a")); + return Progress.DONE; + } + + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java new file mode 100644 index 00000000000..4af0b148164 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTestBase.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.jdisc; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.core.document.ContainerDocumentConfig; +import com.yahoo.container.jdisc.messagebus.MbusServerProvider; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.docproc.AbstractConcreteDocumentFactory; +import com.yahoo.docproc.CallStack; +import com.yahoo.docproc.DocprocService; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext; + +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.jdisc.AbstractResource; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.application.ContainerBuilder; +import com.yahoo.messagebus.Protocol; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.jdisc.MbusClient; +import com.yahoo.messagebus.jdisc.test.RemoteServer; +import com.yahoo.messagebus.jdisc.test.ServerTestDriver; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.shared.SharedSourceSession; +import org.junit.After; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public abstract class DocumentProcessingHandlerTestBase { + + protected DocumentProcessingHandler handler; + protected ServerTestDriver driver; + protected RemoteServer remoteServer; + protected DocumentTypeManager documentTypeManager = new DocumentTypeManager(); + SessionCache sessionCache; + private List<MbusServerProvider> serviceProviders = new ArrayList<>(); + + @Before + public void createHandler() { + documentTypeManager.register(getType()); + + Protocol protocol = new DocumentProtocol(documentTypeManager); + + driver = ServerTestDriver.newInactiveInstanceWithProtocol(protocol); + + sessionCache = + new SessionCache("raw:", driver.client().slobrokId(), "test", "raw:", null, "raw:", documentTypeManager); + + ContainerBuilder builder = driver.parent().newContainerBuilder(); + ComponentRegistry<DocprocService> registry = new ComponentRegistry<>(); + + handler = new DocumentProcessingHandler(registry, + new ComponentRegistry<>(), + new ComponentRegistry<>(), + new DocumentProcessingHandlerParameters(). + setDocumentTypeManager(documentTypeManager). + setContainerDocumentConfig(new ContainerDocumentConfig(new ContainerDocumentConfig.Builder()))); + builder.serverBindings().bind("mbus://*/*", handler); + + ReferencedResource<SharedSourceSession> sessionRef = sessionCache.retainSource(new SourceSessionParams()); + MbusClient sourceClient = new MbusClient(sessionRef.getResource()); + builder.clientBindings().bind("mbus://*/source", sourceClient); + builder.clientBindings().bind("mbus://*/" + MbusRequestContext.internalNoThrottledSource, sourceClient); + sourceClient.start(); + + List<Pair<String, CallStack>> callStacks = getCallStacks(); + List<AbstractResource> resources = new ArrayList<>(); + for (Pair<String, CallStack> callStackPair : callStacks) { + DocprocService service = new DocprocService(callStackPair.getFirst()); + service.setCallStack(callStackPair.getSecond()); + service.setInService(true); + + ComponentId serviceId = new ComponentId(service.getName()); + registry.register(serviceId, service); + + ComponentId sessionName = ComponentId.fromString("chain." + serviceId); + MbusServerProvider serviceProvider = new MbusServerProvider(sessionName, sessionCache, driver.parent()); + serviceProvider.get().start(); + + serviceProviders.add(serviceProvider); + + MbusClient intermediateClient = new MbusClient(serviceProvider.getSession()); + builder.clientBindings().bind("mbus://*/" + sessionName.stringValue(), intermediateClient); + intermediateClient.start(); + resources.add(intermediateClient); + } + + driver.parent().activateContainer(builder); + sessionRef.getReference().close(); + sourceClient.release(); + + for (AbstractResource resource : resources) { + resource.release(); + } + + remoteServer = RemoteServer.newInstance(driver.client().slobrokId(), "foobar", protocol); + } + + @After + public void destroy() { + for (MbusServerProvider serviceProvider : serviceProviders) { + serviceProvider.deconstruct(); + } + driver.close(); + remoteServer.close(); + } + + protected abstract List<Pair<String, CallStack>> getCallStacks(); + + protected abstract DocumentType getType(); + + public boolean sendMessage(String destinationChainName, DocumentMessage msg) { + msg.setRoute(Route.parse("test/chain." + destinationChainName + " " + remoteServer.connectionSpec())); + msg.setPriority(DocumentProtocol.Priority.HIGH_1); + msg.setLoadType(LoadType.DEFAULT); + msg.getTrace().setLevel(9); + msg.setTimeRemaining(60 * 1000); + return driver.client().sendMessage(msg).isAccepted(); + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTransformingMessagesTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTransformingMessagesTestCase.java new file mode 100644 index 00000000000..f3b80506021 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingHandlerTransformingMessagesTestCase.java @@ -0,0 +1,237 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.jdisc; + +import com.yahoo.collections.Pair; +import com.yahoo.docproc.CallStack; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.*; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.update.FieldUpdate; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.Routable; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentProcessingHandlerTransformingMessagesTestCase extends DocumentProcessingHandlerTestBase { + + private static final String FOOBAR = "foobar"; + private final DocumentType type; + + public DocumentProcessingHandlerTransformingMessagesTestCase() { + type = new DocumentType("foo"); + type.addField("foostring", DataType.STRING); + } + + @Override + public List<Pair<String, CallStack>> getCallStacks() { + CallStack stack = new CallStack(); + stack.addLast(new TransformingDocumentProcessor()); + + ArrayList<Pair<String, CallStack>> stacks = new ArrayList<>(1); + stacks.add(new Pair<>(FOOBAR, stack)); + return stacks; + } + + @Override + public DocumentType getType() { + return type; + } + + private Routable sendMessageAndGetResult(DocumentMessage message) throws InterruptedException { + assertTrue(sendMessage(FOOBAR, message)); + + Message result = remoteServer.awaitMessage(60, TimeUnit.SECONDS); + assertNotNull(result); + remoteServer.ackMessage(result); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + assertFalse(reply.hasErrors()); + + return result; + } + + @Test + public void testAllMessages() throws InterruptedException { + put(); + remove(); + update(); + batchDocumentUpdate(); + } + + private void put() throws InterruptedException { + { + PutDocumentMessage message = new PutDocumentMessage(new DocumentPut(new Document(getType(), "doc:nodocstatus:put:to:put"))); + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(PutDocumentMessage.class)); + PutDocumentMessage outputMsg = (PutDocumentMessage)result; + assertThat(outputMsg.getDocumentPut().getDocument().getFieldValue("foostring").toString(), is("banana")); + } + { + PutDocumentMessage message = new PutDocumentMessage(new DocumentPut(new Document(getType(), "doc:nodocstatus:put:to:remove"))); + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(RemoveDocumentMessage.class)); + RemoveDocumentMessage outputMsg = (RemoveDocumentMessage)result; + assertThat(outputMsg.getDocumentId().toString(), is("doc:nodocstatus:put:to:remove")); + } + { + PutDocumentMessage message = new PutDocumentMessage(new DocumentPut(new Document(getType(), "doc:nodocstatus:put:to:update"))); + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(UpdateDocumentMessage.class)); + UpdateDocumentMessage outputMsg = (UpdateDocumentMessage)result; + assertThat(outputMsg.getDocumentUpdate().getId().toString(), is("doc:nodocstatus:put:to:update")); + } + { + PutDocumentMessage message = new PutDocumentMessage(new DocumentPut(new Document(getType(), "doc:nodocstatus:put:to:nothing"))); + assertTrue(sendMessage(FOOBAR, message)); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + assertThat(reply, instanceOf(DocumentReply.class)); + assertFalse(reply.hasErrors()); + } + } + + private void remove() throws InterruptedException { + { + RemoveDocumentMessage message = new RemoveDocumentMessage(new DocumentId("doc:nodocstatus:remove:to:put")); + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(PutDocumentMessage.class)); + PutDocumentMessage outputMsg = (PutDocumentMessage)result; + assertThat(outputMsg.getDocumentPut().getDocument().getId().toString(), is("doc:nodocstatus:remove:to:put")); + } + + { + RemoveDocumentMessage message = new RemoveDocumentMessage(new DocumentId("doc:nodocstatus:remove:to:remove")); + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(RemoveDocumentMessage.class)); + RemoveDocumentMessage outputMsg = (RemoveDocumentMessage)result; + assertThat(outputMsg.getDocumentId().toString(), is("doc:nodocstatus:remove:to:remove")); + } + { + RemoveDocumentMessage message = new RemoveDocumentMessage(new DocumentId("doc:nodocstatus:remove:to:update")); + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(UpdateDocumentMessage.class)); + UpdateDocumentMessage outputMsg = (UpdateDocumentMessage)result; + assertThat(outputMsg.getDocumentUpdate().getId().toString(), is("doc:nodocstatus:remove:to:update")); + } + { + RemoveDocumentMessage message = new RemoveDocumentMessage(new DocumentId("doc:nodocstatus:remove:to:nothing")); + assertTrue(sendMessage(FOOBAR, message)); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + assertThat(reply, instanceOf(DocumentReply.class)); + assertFalse(reply.hasErrors()); + } + } + + private void update() throws InterruptedException { + { + UpdateDocumentMessage message = new UpdateDocumentMessage(new DocumentUpdate(getType(), "doc:nodocstatus:update:to:put")); + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(PutDocumentMessage.class)); + PutDocumentMessage outputMsg = (PutDocumentMessage)result; + assertThat(outputMsg.getDocumentPut().getDocument().getId().toString(), is("doc:nodocstatus:update:to:put")); + } + + { + UpdateDocumentMessage message = new UpdateDocumentMessage(new DocumentUpdate(getType(), "doc:nodocstatus:update:to:remove")); + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(RemoveDocumentMessage.class)); + RemoveDocumentMessage outputMsg = (RemoveDocumentMessage)result; + assertThat(outputMsg.getDocumentId().toString(), is("doc:nodocstatus:update:to:remove")); + } + { + UpdateDocumentMessage message = new UpdateDocumentMessage(new DocumentUpdate(getType(), "doc:nodocstatus:update:to:update")); + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(UpdateDocumentMessage.class)); + UpdateDocumentMessage outputMsg = (UpdateDocumentMessage)result; + assertThat(outputMsg.getDocumentUpdate().getId().toString(), is("doc:nodocstatus:update:to:update")); + } + { + UpdateDocumentMessage message = new UpdateDocumentMessage(new DocumentUpdate(getType(), "doc:nodocstatus:update:to:nothing")); + assertTrue(sendMessage(FOOBAR, message)); + Reply reply = driver.client().awaitReply(60, TimeUnit.SECONDS); + assertNotNull(reply); + assertThat(reply, instanceOf(DocumentReply.class)); + assertFalse(reply.hasErrors()); + } + } + + private void batchDocumentUpdate() throws InterruptedException { + DocumentUpdate doc1 = new DocumentUpdate(getType(), new DocumentId("userdoc:test:12345:batch:nodocstatus:keep:this")); + DocumentUpdate doc2 = new DocumentUpdate(getType(), new DocumentId("userdoc:test:12345:batch:nodocstatus:skip:this")); + + Field testField = getType().getField("foostring"); + doc1.addFieldUpdate(FieldUpdate.createAssign(testField, new StringFieldValue("1 not yet processed"))); + doc2.addFieldUpdate(FieldUpdate.createAssign(testField, new StringFieldValue("2 not yet processed"))); + + BatchDocumentUpdateMessage message = new BatchDocumentUpdateMessage(12345); + message.addUpdate(doc1); + message.addUpdate(doc2); + + Routable result = sendMessageAndGetResult(message); + assertThat(result, instanceOf(UpdateDocumentMessage.class)); + DocumentUpdate outputUpd = ((UpdateDocumentMessage)result).getDocumentUpdate(); + assertThat(outputUpd.getId().toString(), is("userdoc:test:12345:batch:nodocstatus:keep:this")); + } + + public class TransformingDocumentProcessor extends DocumentProcessor { + + @Override + public Progress process(Processing processing) { + ListIterator<DocumentOperation> it = processing.getDocumentOperations().listIterator(); + while (it.hasNext()) { + DocumentOperation op = it.next(); + String id = op.getId().toString(); + if ("doc:nodocstatus:put:to:put".equals(id)) { + Document doc = ((DocumentPut)op).getDocument(); + doc.setFieldValue("foostring", new StringFieldValue("banana")); + } else if ("doc:nodocstatus:put:to:remove".equals(id)) { + it.set(new DocumentRemove(new DocumentId(id))); + } else if ("doc:nodocstatus:put:to:update".equals(id)) { + it.set(new DocumentUpdate(getType(), id)); + } else if ("doc:nodocstatus:put:to:nothing".equals(id)) { + it.remove(); + } else if ("doc:nodocstatus:remove:to:put".equals(id)) { + it.set(new DocumentPut(getType(), op.getId())); + } else if ("doc:nodocstatus:remove:to:remove".equals(id)) { + //nada + } else if ("doc:nodocstatus:remove:to:update".equals(id)) { + it.set(new DocumentUpdate(getType(), id)); + } else if ("doc:nodocstatus:remove:to:nothing".equals(id)) { + it.remove(); + } else if ("doc:nodocstatus:update:to:put".equals(id)) { + it.set(new DocumentPut(getType(), op.getId())); + } else if ("doc:nodocstatus:update:to:remove".equals(id)) { + it.set(new DocumentRemove(new DocumentId(id))); + } else if ("doc:nodocstatus:update:to:update".equals(id)) { + //nada + } else if ("doc:nodocstatus:update:to:nothing".equals(id)) { + it.remove(); + } else if ("userdoc:12345:6789:multiop:nodocstatus:keep:this".equals(id)) { + //nada + } else if ("userdoc:12345:6789:multiop:nodocstatus:skip:this".equals(id)) { + it.remove(); + } else if ("userdoc:test:12345:batch:nodocstatus:keep:this".equals(id)) { + //nada + } else if ("userdoc:test:12345:batch:nodocstatus:skip:this".equals(id)) { + it.remove(); + } + } + return Progress.DONE; + } + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java new file mode 100644 index 00000000000..8dc811bece6 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.jdisc; + +import com.yahoo.docproc.Processing; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import org.junit.Test; + +import java.net.URI; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.PriorityBlockingQueue; + +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentProcessingTaskPrioritizationTestCase { + + @Test + public void proritization() { + Queue<DocumentProcessingTask> queue = new PriorityBlockingQueue<>(); + + DocumentProcessingTask highest = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGHEST); + DocumentProcessingTask veryhigh = new TestDocumentProcessingTask(DocumentProtocol.Priority.VERY_HIGH); + DocumentProcessingTask high1 = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGH_1); + DocumentProcessingTask normal_1 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); + DocumentProcessingTask low_1 = new TestDocumentProcessingTask(DocumentProtocol.Priority.LOW_1); + DocumentProcessingTask verylow = new TestDocumentProcessingTask(DocumentProtocol.Priority.VERY_LOW); + DocumentProcessingTask lowest = new TestDocumentProcessingTask(DocumentProtocol.Priority.LOWEST); + + DocumentProcessingTask normal_2 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); + DocumentProcessingTask normal_3 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); + DocumentProcessingTask normal_4 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); + + DocumentProcessingTask highest_2 = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGHEST); + DocumentProcessingTask highest_3 = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGHEST); + + + queue.add(highest); + queue.add(veryhigh); + queue.add(high1); + queue.add(normal_1); + queue.add(low_1); + queue.add(verylow); + queue.add(lowest); + + queue.add(normal_2); + queue.add(normal_3); + queue.add(normal_4); + + queue.add(highest_2); + queue.add(highest_3); + + assertThat(queue.poll(), sameInstance(highest)); + assertThat(queue.poll(), sameInstance(highest_2)); + assertThat(queue.poll(), sameInstance(highest_3)); + assertThat(queue.poll(), sameInstance(veryhigh)); + assertThat(queue.poll(), sameInstance(high1)); + assertThat(queue.poll(), sameInstance(normal_1)); + assertThat(queue.poll(), sameInstance(normal_2)); + assertThat(queue.poll(), sameInstance(normal_3)); + assertThat(queue.poll(), sameInstance(normal_4)); + assertThat(queue.poll(), sameInstance(low_1)); + assertThat(queue.poll(), sameInstance(verylow)); + assertThat(queue.poll(), sameInstance(lowest)); + assertThat(queue.poll(), nullValue()); + } + + private class TestDocumentProcessingTask extends DocumentProcessingTask { + private TestDocumentProcessingTask(DocumentProtocol.Priority priority) { + super(new TestRequestContext(priority), null, null); + } + } + + private class TestRequestContext implements RequestContext { + private final DocumentProtocol.Priority priority; + + public TestRequestContext(DocumentProtocol.Priority priority) { + this.priority = priority; + } + + @Override + public List<Processing> getProcessings() { + return null; + } + + @Override + public void skip() { + } + + @Override + public void processingDone(List<Processing> processing) { + } + + @Override + public void processingFailed(ErrorCode error, String msg) { + } + + @Override + public void processingFailed(Exception exception) { + } + + @Override + public int getApproxSize() { + return 0; + } + + @Override + public int getPriority() { + return priority.getValue(); + } + + @Override + public boolean isProcessable() { + return true; + } + + @Override + public URI getUri() { + return null; + } + + @Override + public String getServiceName() { + return null; + } + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java new file mode 100644 index 00000000000..e5d3a4dc137 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/proxy/SchemaMappingAndAccessesTest.java @@ -0,0 +1,566 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.proxy; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import com.yahoo.collections.Pair; +import com.yahoo.config.docproc.SchemamappingConfig; +import com.yahoo.docproc.Accesses; +import com.yahoo.docproc.Accesses.Field; +import com.yahoo.docproc.Call; +import com.yahoo.docproc.DocumentProcessingAbstractTestCase.TestDocumentProcessor1; +import com.yahoo.docproc.DocumentProcessor; +import com.yahoo.docproc.Processing; +import com.yahoo.document.DataType; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.StructDataType; +import com.yahoo.document.annotation.Annotation; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.annotation.SpanTree; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.document.datatypes.StructuredFieldValue; +import com.yahoo.document.update.FieldUpdate; + +public class SchemaMappingAndAccessesTest extends junit.framework.TestCase { + + private Document getDoc() { + DocumentType type = new DocumentType("album"); + AnnotationType personType = new AnnotationType("person"); + Annotation person = new Annotation(personType); + type.addField("title", DataType.STRING); + type.addField("artist", DataType.STRING); + type.addField("guitarist", DataType.STRING); + type.addField("year", DataType.INT); + type.addField("labels", DataType.getArray(DataType.STRING)); + Document doc = new Document(type, new DocumentId("doc:map:test:1")); + doc.setFieldValue("title", new StringFieldValue("Black Rock")); + StringFieldValue joe = new StringFieldValue("Joe Bonamassa"); + joe.setSpanTree(new SpanTree("mytree").annotate(person)); + doc.setFieldValue("artist", joe); + doc.setFieldValue("year", new IntegerFieldValue(2010)); + Array<StringFieldValue> labels = new Array<>(type.getField("labels").getDataType()); + labels.add(new StringFieldValue("audun")); + labels.add(new StringFieldValue("tylden")); + doc.setFieldValue("labels", labels); + + StructDataType personStructType = new StructDataType("artist"); + personStructType.addField(new com.yahoo.document.Field("firstname", DataType.STRING)); + personStructType.addField(new com.yahoo.document.Field("lastname", DataType.STRING)); + type.addField("listeners", DataType.getArray(personStructType)); + + Array<Struct> listeners = new Array<>(type.getField("listeners").getDataType()); + + Struct listenerOne = new Struct(personStructType); + listenerOne.setFieldValue("firstname", new StringFieldValue("per")); + listenerOne.setFieldValue("lastname", new StringFieldValue("olsen")); + Struct listenerTwo = new Struct(personStructType); + listenerTwo.setFieldValue("firstname", new StringFieldValue("anders")); + listenerTwo.setFieldValue("lastname", new StringFieldValue("and")); + + listeners.add(listenerOne); + listeners.add(listenerTwo); + + doc.setFieldValue("listeners", listeners); + + return doc; + } + + public void testMappingArrays() { + Document doc = getDoc(); + DocumentProcessor proc = new TestMappingArrayProcessor(); + + Map<String, String> fieldMap = new HashMap<>(); + fieldMap.put("label", "labels[0]"); + ProxyDocument mapped = new ProxyDocument(proc, doc, fieldMap); + + Processing p = Processing.of(new DocumentPut(mapped)); + proc.process(p); + + assertEquals(2, ((Array<StringFieldValue>) doc.getFieldValue("labels")).size()); + assertEquals(new StringFieldValue("EMI"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(0)); + assertEquals(new StringFieldValue("tylden"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(1)); + + + fieldMap.clear(); + fieldMap.put("label", "labels[2]"); + mapped = new ProxyDocument(proc, doc, fieldMap); + + p = Processing.of(new DocumentPut(mapped)); + try { + proc.process(p); + fail("Should not have worked"); + } catch (IllegalArgumentException iae) { + //ok! + } + assertEquals(2, ((Array<StringFieldValue>) doc.getFieldValue("labels")).size()); + assertEquals(new StringFieldValue("EMI"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(0)); + assertEquals(new StringFieldValue("tylden"), ((Array<StringFieldValue>) doc.getFieldValue("labels")).get(1)); + } + + public void testMappingStructsInArrays() { + Document doc = getDoc(); + DocumentProcessor proc = new TestMappingStructInArrayProcessor(); + + Map<String, String> fieldMap = new HashMap<>(); + fieldMap.put("name", "listeners[0].firstname"); + ProxyDocument mapped = new ProxyDocument(proc, doc, fieldMap); + + Processing p = Processing.of(new DocumentPut(mapped)); + proc.process(p); + + assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size()); + assertEquals("peter", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString())); + assertEquals("olsen", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString())); + assertEquals("anders", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString())); + assertEquals("and", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")).getString())); + + + fieldMap.clear(); + fieldMap.put("name", "listeners[2].firstname"); + mapped = new ProxyDocument(proc, doc, fieldMap); + + p = Processing.of(new DocumentPut(mapped)); + try { + proc.process(p); + fail("Should not have worked"); + } catch (IllegalArgumentException iae) { + //ok! + } + assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size()); + assertEquals("peter", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString())); + assertEquals("olsen", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString())); + assertEquals("anders", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString())); + assertEquals("and", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")).getString())); + + + //test remove: + proc = new TestRemovingMappingStructInArrayProcessor(); + + fieldMap.clear(); + fieldMap.put("name", "listeners[1].lastname"); + mapped = new ProxyDocument(proc, doc, fieldMap); + + p = Processing.of(new DocumentPut(mapped)); + proc.process(p); + + assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size()); + assertEquals("peter", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString())); + assertEquals("olsen", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString())); + assertEquals("anders", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString())); + assertNull(((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")); + + + fieldMap.clear(); + fieldMap.put("name", "listeners[2].lastname"); + mapped = new ProxyDocument(proc, doc, fieldMap); + + p = Processing.of(new DocumentPut(mapped)); + try { + proc.process(p); + fail("Should not have worked"); + } catch (IllegalArgumentException iae) { + //ok! + } + assertEquals(2, ((Array<Struct>) doc.getFieldValue("listeners")).size()); + assertEquals("peter", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("firstname")).getString())); + assertEquals("olsen", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(0).getFieldValue("lastname")).getString())); + assertEquals("anders", (((StringFieldValue)((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("firstname")).getString())); + assertNull(((Array<Struct>) doc.getFieldValue("listeners")).get(1).getFieldValue("lastname")); + + } + + + public void testMappingSpanTrees() { + Document doc = getDoc(); + Map<String, String> fieldMap = new HashMap<>(); + fieldMap.put("t", "title"); + fieldMap.put("a", "artist"); + fieldMap.put("g", "guitarist"); + ProxyDocument mapped = new ProxyDocument(new TestDocumentProcessor1(), doc, fieldMap); + Iterator<SpanTree> itSpanTreesDoc = ((StringFieldValue) doc.getFieldValue("artist")).getSpanTrees().iterator(); + Iterator<Annotation> itAnnotDoc = itSpanTreesDoc.next().iterator(); + Iterator<SpanTree> itSpanTreesMapped = ((StringFieldValue) mapped.getFieldValue("artist")).getSpanTrees().iterator(); + Iterator<Annotation> itAnnotMapped = itSpanTreesMapped.next().iterator(); + + assertEquals(itAnnotDoc.next().getType().getName(), "person"); + assertFalse(itAnnotDoc.hasNext()); + assertEquals(itAnnotMapped.next().getType().getName(), "person"); + assertFalse(itAnnotMapped.hasNext()); + + AnnotationType guitaristType = new AnnotationType("guitarist"); + Annotation guitarist = new Annotation(guitaristType); + StringFieldValue bona = new StringFieldValue("Bonamassa"); + bona.setSpanTree(new SpanTree("mytree").annotate(guitarist)); + StringFieldValue clapton = new StringFieldValue("Clapton"); + mapped.setFieldValue("a", bona); + mapped.setFieldValue("g", clapton); + + itSpanTreesDoc = ((StringFieldValue) doc.getFieldValue("artist")).getSpanTrees().iterator(); + itAnnotDoc = itSpanTreesDoc.next().iterator(); + itSpanTreesMapped = ((StringFieldValue) mapped.getFieldValue("artist")).getSpanTrees().iterator(); + itAnnotMapped = itSpanTreesMapped.next().iterator(); + + assertEquals(itAnnotDoc.next().getType().getName(), "guitarist"); + assertFalse(itAnnotDoc.hasNext()); + assertEquals(itAnnotMapped.next().getType().getName(), "guitarist"); + assertFalse(itAnnotMapped.hasNext()); + + assertSame(((StringFieldValue) doc.getFieldValue("artist")).getSpanTrees().iterator().next(), ((StringFieldValue) mapped.getFieldValue("a")).getSpanTrees().iterator().next()); + //assertSame(clapton, mapped.getFieldValue("g")); + //assertSame(bona, mapped.getFieldValue("a")); + } + + public void testMappedDoc() { + Document doc = getDoc(); + Map<String, String> fieldMap = new HashMap<>(); + fieldMap.put("t", "title"); + fieldMap.put("a", "artist"); + ProxyDocument mapped = new ProxyDocument(new TestDocumentProcessor1(), doc, fieldMap); + //Document mapped=doc; + //mapped.setFieldMap(fieldMap); + assertEquals(new StringFieldValue("Black Rock"), mapped.getFieldValue("t")); + //assertEquals(new StringFieldValue("Black Rock"), proxy.getFieldValue(new com.yahoo.document.Field("t"))); + assertEquals(new StringFieldValue("Joe Bonamassa").getWrappedValue(), mapped.getFieldValue("a").getWrappedValue()); + mapped.setFieldValue("t", new StringFieldValue("The Ballad Of John Henry")); + StringFieldValue bona = new StringFieldValue("Bonamassa"); + mapped.setFieldValue("a", bona); + //mapped.setFieldValue("a", new StringFieldValue("Bonamassa")); + assertEquals(new StringFieldValue("The Ballad Of John Henry"), doc.getFieldValue("title")); + assertEquals(new StringFieldValue("The Ballad Of John Henry"), mapped.getFieldValue("t")); + assertEquals(new StringFieldValue("Bonamassa"), doc.getFieldValue("artist")); + assertEquals(new StringFieldValue("Bonamassa"), mapped.getFieldValue("a")); + mapped.setFieldValue("a", mapped.getFieldValue("a") + "Hughes"); + assertEquals(new StringFieldValue("BonamassaHughes"), mapped.getFieldValue("a")); + // Verify consistency when using string values to manipluate annotation span trees + StringFieldValue unmapped1 = (StringFieldValue) doc.getFieldValue("artist"); + StringFieldValue unmapped2 = (StringFieldValue) doc.getFieldValue("artist"); + assertTrue(unmapped1==unmapped2); + unmapped1.setSpanTree(new SpanTree("test")); + assertEquals(unmapped2.getSpanTree("test").getName(), "test"); + + StringFieldValue mapped1 = (StringFieldValue) mapped.getFieldValue("a"); + mapped1.setSpanTree(new SpanTree("test2")); + StringFieldValue mapped2 = (StringFieldValue) mapped.getFieldValue("a"); + assertTrue(mapped1==mapped2); + assertEquals(mapped2.getSpanTree("test2").getName(), "test2"); + + mapped.removeFieldValue("a"); + assertEquals(mapped.getFieldValue("a"), null); + mapped.removeFieldValue(mapped.getField("t")); + assertEquals(mapped.getFieldValue("t"), null); + mapped.setFieldValue("a", new StringFieldValue("Bonamassa")); + assertEquals(new StringFieldValue("Bonamassa"), doc.getFieldValue("artist")); + mapped.removeFieldValue("a"); + assertEquals(mapped.getFieldValue("a"), null); + } + + public void testMappedDocAPI() { + Document doc = getDoc(); + Map<String, String> fieldMap = new HashMap<>(); + fieldMap.put("t", "title"); + fieldMap.put("a", "artist"); + ProxyDocument mapped = new ProxyDocument(new TestDocumentProcessor1(), doc, fieldMap); + assertEquals(mapped.getFieldValue("title"), doc.getFieldValue("title")); + assertEquals(mapped.getFieldValue(new com.yahoo.document.Field("title")), doc.getFieldValue((new com.yahoo.document.Field("title")))); + mapped.setFieldValue("title", "foo"); + assertEquals(doc.getFieldValue("title").getWrappedValue(), "foo"); + assertEquals(mapped.getWrappedDocumentOperation().getId().toString(), "doc:map:test:1"); + assertEquals(doc, mapped); + assertEquals(doc.toString(), mapped.toString()); + assertEquals(doc.hashCode(), mapped.hashCode()); + assertEquals(doc.clone(), mapped.clone()); + assertEquals(doc.iterator().hasNext(), mapped.iterator().hasNext()); + assertEquals(doc.getId(), mapped.getId()); + assertEquals(doc.getDataType(), mapped.getDataType()); + mapped.setLastModified(56l); + assertEquals(doc.getLastModified(), (Long)56l); + assertEquals(mapped.getLastModified(), (Long)56l); + mapped.setId(new DocumentId("doc:map:test:2")); + assertEquals(mapped.getId().toString(), "doc:map:test:2"); + assertEquals(doc.getId().toString(), "doc:map:test:2"); + assertEquals(doc.getHeader(), mapped.getHeader()); + assertEquals(doc.getBody(), mapped.getBody()); + assertEquals(doc.getSerializedSize(), mapped.getSerializedSize()); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ByteArrayOutputStream bos2 = new ByteArrayOutputStream(); + mapped.serialize(bos); + doc.serialize(bos2); + assertEquals(bos.toString(), bos2.toString()); + assertEquals(mapped.toXml(), doc.toXml()); + assertEquals(mapped.getFieldCount(), doc.getFieldCount()); + assertTrue(mapped.getDocument()==doc); + + mapped.clear(); + assertNull(mapped.getFieldValue("title")); + assertNull(doc.getFieldValue("title")); + mapped.setDataType(new DocumentType("newType")); + assertEquals(doc.getDataType().getName(), "newType"); + } + + public void testMappedDocUpdateAPI() { + Document doc = getDoc(); + DocumentType type = doc.getDataType(); + DocumentUpdate dud = new DocumentUpdate(type, new DocumentId("doc:map:test:1")); + FieldUpdate assignSingle = FieldUpdate.createAssign(type.getField("title"), new StringFieldValue("something")); + Map<String, String> fieldMap = new HashMap<>(); + fieldMap.put("t", "title"); + fieldMap.put("a", "artist"); + ProxyDocumentUpdate pup = new ProxyDocumentUpdate(dud, fieldMap); + pup.addFieldUpdate(assignSingle); + assertEquals(pup.getFieldUpdates(), dud.getFieldUpdates()); + assertEquals(pup.getDocumentType(), dud.getDocumentType()); + assertEquals(pup.getFieldUpdate(new com.yahoo.document.Field("title")).size(), 1); + assertEquals(pup.getFieldUpdate(0), dud.getFieldUpdate(0)); + assertEquals(pup.getFieldUpdate("title"), dud.getFieldUpdate("title")); + assertEquals(pup.getId(), dud.getId()); + assertEquals(pup.getType(), dud.getType()); + assertEquals(pup.applyTo(doc), dud); + assertEquals(doc.getFieldValue("title").getWrappedValue(), "something"); + assertEquals(pup, dud); + assertEquals(pup.hashCode(), dud.hashCode()); + assertEquals(pup.toString(), dud.toString()); + assertEquals(pup.size(), dud.size()); + assertEquals(pup.getWrappedDocumentOperation().getId().toString(), "doc:map:test:1"); + } + + public void testMappedDocStruct() { + StructDataType materialsStructType = new StructDataType("materialstype"); + materialsStructType.addField(new com.yahoo.document.Field("ceiling", DataType.STRING)); + materialsStructType.addField(new com.yahoo.document.Field("walls", DataType.STRING)); + + DocumentType docType = new DocumentType("album"); + docType.addField("title", DataType.STRING); + docType.addField("artist", DataType.STRING); + StructDataType storeStructType = new StructDataType("storetype"); + storeStructType.addField(new com.yahoo.document.Field("name", DataType.STRING)); + storeStructType.addField(new com.yahoo.document.Field("city", DataType.STRING)); + storeStructType.addField(new com.yahoo.document.Field("materials", materialsStructType)); + docType.addField("store", storeStructType); + + Document doc = new Document(docType, new DocumentId("doc:map:test:1")); + doc.setFieldValue("title", new StringFieldValue("Black Rock")); + doc.setFieldValue("artist", new StringFieldValue("Joe Bonamassa")); + Struct material = new Struct(materialsStructType); + material.setFieldValue("ceiling", new StringFieldValue("wood")); + material.setFieldValue("walls", new StringFieldValue("brick")); + Struct store = new Struct(storeStructType); + store.setFieldValue("name", new StringFieldValue("Platekompaniet")); + store.setFieldValue("city", new StringFieldValue("Trondheim")); + store.setFieldValue(storeStructType.getField("materials"), material); + doc.setFieldValue(docType.getField("store"), store); + + Map<String, String> fieldMap = new HashMap<>(); + fieldMap.put("t", "title"); + fieldMap.put("c", "store.city"); + fieldMap.put("w", "store.materials.walls"); + ProxyDocument mapped = new ProxyDocument(new TestDocumentProcessor1(), doc, fieldMap); + assertEquals(new StringFieldValue("Trondheim"), mapped.getFieldValue("c")); + assertEquals(new StringFieldValue("Black Rock"), mapped.getFieldValue("t")); + assertEquals(new StringFieldValue("brick"), mapped.getFieldValue("w")); + assertEquals(new StringFieldValue("brick"), material.getFieldValue("walls")); + mapped.setFieldValue("c", new StringFieldValue("Steinkjer")); + mapped.setFieldValue("w", new StringFieldValue("plaster")); + assertEquals(new StringFieldValue("plaster"), mapped.getFieldValue("w")); + assertEquals(new StringFieldValue("plaster"), material.getFieldValue("walls")); + assertEquals(new StringFieldValue("Steinkjer"), store.getFieldValue("city")); + assertEquals(new StringFieldValue("Steinkjer"), mapped.getFieldValue("c")); + assertEquals(new StringFieldValue("Steinkjer"), mapped.getFieldValue("c")); + mapped.setFieldValue("c", new StringFieldValue("Levanger")); + assertEquals(new StringFieldValue("Levanger"), store.getFieldValue("city")); + assertEquals(new StringFieldValue("Levanger"), mapped.getFieldValue("c")); + mapped.setFieldValue("c", mapped.getFieldValue("c") + "Kommune"); + assertEquals(new StringFieldValue("LevangerKommune"), mapped.getFieldValue("c")); + //mapped.set(mapped.getField("c"), mapped.get("c")+"Styre"); + //assertEquals(new StringFieldValue("LevangerKommuneStyre"), mapped.getFieldValue("c")); + } + + public void testSchemaMap() { + SchemaMap map = new SchemaMap(); + map.addMapping("mychain", "com.yahoo.MyDocProc", "mydoctype", "inDoc1", "inProc1"); + map.addMapping("mychain", "com.yahoo.MyDocProc", "mydoctype", "inDoc2", "inProc2"); + Map<Pair<String, String>, String> cMap = map.chainMap("mychain", "com.yahoo.MyDocProc"); + assertEquals("inDoc1", cMap.get(new Pair<>("mydoctype", "inProc1"))); + assertEquals("inDoc2", cMap.get(new Pair<>("mydoctype", "inProc2"))); + assertNull(cMap.get(new Pair<>("invalidtype", "inProc2"))); + Map<Pair<String, String>, String> noMap = map.chainMap("invalidchain", "com.yahoo.MyDocProc"); + Map<Pair<String, String>, String> noMap2 = map.chainMap("mychain", "com.yahoo.MyInvalidDocProc"); + assertTrue(noMap.isEmpty()); + assertTrue(noMap2.isEmpty()); + + DocumentProcessor proc = new TestDocumentProcessor1(); + proc.setFieldMap(cMap); + Map<String, String> dMap = proc.getDocMap("mydoctype"); + assertEquals("inDoc1", dMap.get("inProc1")); + assertEquals("inDoc2", dMap.get("inProc2")); + } + + public void testSchemaMapKey() { + SchemaMap map = new SchemaMap(null); + SchemaMap.SchemaMapKey key1 = map.new SchemaMapKey("chain", "docproc", "doctype", "from"); + SchemaMap.SchemaMapKey key1_1 = map.new SchemaMapKey("chain", "docproc", "doctype", "from"); + SchemaMap.SchemaMapKey key2 = map.new SchemaMapKey("chain", "docproc", "doctype2", "from"); + assertTrue(key1.equals(key1_1)); + assertFalse(key1.equals(key2)); + } + + public void testSchemaMapConfig() { + SchemaMap map = new SchemaMap(null); + SchemamappingConfig.Builder scb = new SchemamappingConfig.Builder(); + scb.fieldmapping(new SchemamappingConfig.Fieldmapping.Builder().chain("mychain").docproc("mydocproc").doctype("mydoctype"). + indocument("myindoc").inprocessor("myinprocessor")); + map.configure(new SchemamappingConfig(scb)); + assertEquals(map.chainMap("mychain", "mydocproc").get(new Pair<>("mydoctype", "myinprocessor")), "myindoc"); + } + + public void testSchemaMapNoDocType() { + SchemaMap map = new SchemaMap(null); + map.addMapping("mychain", "com.yahoo.MyDocProc", null, "inDoc1", "inProc1"); + map.addMapping("mychain", "com.yahoo.MyDocProc", null, "inDoc2", "inProc2"); + Map<Pair<String, String>, String> cMap = map.chainMap("mychain", "com.yahoo.MyDocProc"); + DocumentProcessor proc = new TestDocumentProcessor1(); + proc.setFieldMap(cMap); + Map<String, String> dMap = proc.getDocMap("mydoctype"); + assertEquals("inDoc1", dMap.get("inProc1")); + assertEquals("inDoc2", dMap.get("inProc2")); + } + + public void testProxyAndSecure() { + DocumentProcessor procOK = new TestDPSecure(); + Map<Pair<String, String>, String> fieldMap = new HashMap<>(); + fieldMap.put(new Pair<>("album", "titleMapped"), "title"); + procOK.setFieldMap(fieldMap); + DocumentPut put = new DocumentPut(getDoc()); + Document proxyDoc = new Call(procOK).configDoc(procOK, put).getDocument(); + procOK.process(Processing.of(new DocumentPut(proxyDoc))); + assertEquals(proxyDoc.getFieldValue("title").toString(), "MyTitle MyTitle"); + } + + public void testProxyAndSecureSecureFailing() { + DocumentProcessor procInsecure = new TestDPInsecure(); + Map<Pair<String, String>, String> fieldMap = new HashMap<>(); + fieldMap.put(new Pair<>("album", "titleMapped"), "title"); + procInsecure.setFieldMap(fieldMap); + DocumentPut put = new DocumentPut(getDoc()); + Document doc = new Call(procInsecure).configDoc(procInsecure, put).getDocument(); + try { + procInsecure.process(Processing.of(new DocumentPut(doc))); + fail("Insecure docproc went through"); + } catch (Exception e) { + assertTrue(e.getMessage().matches(".*allowed.*")); + } + //assertEquals(doc.get("title"), "MyTitle"); + } + + /** + * To make it less likely to break schema mapping, we enforce that ProxyDocument does wrap every public + * non-static, non-final method on Document and StructuredFieldValue + */ + public void testVerifyProxyDocumentOverridesEverything() { + List<Method> allPublicFromProxyDocument = new ArrayList<>(); + for (Method m : ProxyDocument.class.getDeclaredMethods()) { + if (Modifier.isPublic(m.getModifiers())) { + allPublicFromProxyDocument.add(m); + } + } + List<Method> allPublicFromDoc = new ArrayList<>(); + for (Method m : Document.class.getDeclaredMethods()) { + if (mustBeOverriddenInProxyDocument(m)) { + allPublicFromDoc.add(m); + } + } + for (Method m : StructuredFieldValue.class.getDeclaredMethods()) { + if (mustBeOverriddenInProxyDocument(m)) { + allPublicFromDoc.add(m); + } + } + + for (Method m : allPublicFromDoc) { + boolean thisOneOk=false; + for (Method pdM : allPublicFromProxyDocument) { + if (sameNameAndParams(m, pdM)) thisOneOk=true; + } + if (!thisOneOk) { + throw new RuntimeException("ProxyDocument must override all public methods from Document. " + + "Missing: '"+m+"'. If the method doesn't need field mapping or @Accesses check, just " + + "override it and delegate the call to 'doc'."); + + } + } + } + + private boolean mustBeOverriddenInProxyDocument(Method m) { + if (!Modifier.isPublic(m.getModifiers())) return false; + if (Modifier.isStatic(m.getModifiers())) return false; + if (Modifier.isFinal(m.getModifiers())) return false; + return true; + } + + private boolean sameNameAndParams(Method m1, Method m2) { + if (!m1.getName().equals(m2.getName())) return false; + if (m1.getParameterTypes().length!=m2.getParameterTypes().length) return false; + for (int i = 0; i<m1.getParameterTypes().length; i++) { + if (!m1.getParameterTypes()[i].equals(m2.getParameterTypes()[i])) return false; + } + return true; + } + + @Accesses(value = { @Field(dataType = "String", description = "", name = "titleMapped") }) + public static class TestDPSecure extends DocumentProcessor { + + public Progress process(Processing processing) { + Document document = ((DocumentPut)processing.getDocumentOperations().get(0)).getDocument(); + document.setFieldValue("titleMapped", new StringFieldValue("MyTitle")); + document.setFieldValue("titleMapped", new StringFieldValue(document.getFieldValue("titleMapped").toString() + " MyTitle")); + return Progress.DONE; + } + } + + @Accesses(value = { @Field(dataType = "String", description = "", name = "titleMappedFoo") }) + public static class TestDPInsecure extends DocumentProcessor { + + public Progress process(Processing processing) { + Document document = ((DocumentPut)processing.getDocumentOperations().get(0)).getDocument(); + document.setFieldValue("titleMapped", new StringFieldValue("MyTitle")); + document.setFieldValue("titleMapped", new StringFieldValue(document.getFieldValue("titleMapped").toString() + " MyTitle")); + return Progress.DONE; + } + } + + public static class TestMappingArrayProcessor extends DocumentProcessor { + public Progress process(Processing processing) { + Document document = ((DocumentPut)processing.getDocumentOperations().get(0)).getDocument(); + document.setFieldValue("label", new StringFieldValue("EMI")); + return Progress.DONE; + } + } + + public static class TestMappingStructInArrayProcessor extends DocumentProcessor { + public Progress process(Processing processing) { + Document document = ((DocumentPut)processing.getDocumentOperations().get(0)).getDocument();; + document.setFieldValue("name", new StringFieldValue("peter")); + return Progress.DONE; + } + } + + public static class TestRemovingMappingStructInArrayProcessor extends DocumentProcessor { + public Progress process(Processing processing) { + Document document = ((DocumentPut)processing.getDocumentOperations().get(0)).getDocument();; + document.removeFieldValue("name"); + return Progress.DONE; + } + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/util/SplitterJoinerTestCase.java b/docproc/src/test/java/com/yahoo/docproc/util/SplitterJoinerTestCase.java new file mode 100644 index 00000000000..2d5a61fa005 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/util/SplitterJoinerTestCase.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.docproc.util; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.docproc.SplitterJoinerDocumentProcessorConfig; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.docproc.Processing; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.datatypes.Array; +import com.yahoo.document.datatypes.StringFieldValue; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SplitterJoinerTestCase { + + @Test + public void testSplitJoin() { + ConfigGetter<SplitterJoinerDocumentProcessorConfig> getter = new ConfigGetter<>(SplitterJoinerDocumentProcessorConfig.class); + ConfigGetter<DocumentmanagerConfig> docManGetter = new ConfigGetter<>(DocumentmanagerConfig.class); + + SplitterJoinerDocumentProcessorConfig cfg = + getter.getConfig("file:src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg"); + DocumentmanagerConfig docManCfg = + docManGetter.getConfig("file:src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg"); + + SplitterDocumentProcessor splitter = new SplitterDocumentProcessor(cfg, docManCfg); + + DocumentTypeManager manager = splitter.manager; + + + /**** Create documents: ****/ + + Document inner1 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:one"); + inner1.setFieldValue("name", new StringFieldValue("Donald Duck")); + inner1.setFieldValue("content", new StringFieldValue("Lives in Duckburg")); + Document inner2 = new Document(manager.getDocumentType("docindoc"), "doc:inner:number:two"); + inner2.setFieldValue("name", new StringFieldValue("Uncle Scrooge")); + inner2.setFieldValue("content", new StringFieldValue("Lives in Duckburg, too.")); + + Array<Document> innerArray = (Array<Document>) manager.getDocumentType("outerdoc").getField("innerdocuments").getDataType().createFieldValue(); + innerArray.add(inner1); + innerArray.add(inner2); + + Document outer = new Document(manager.getDocumentType("outerdoc"), "doc:outer:the:only:one"); + outer.setFieldValue("innerdocuments", innerArray); + + /**** End create documents ****/ + + + Processing p = Processing.of(new DocumentPut(outer)); + splitter.process(p); + + assertEquals(2, p.getDocumentOperations().size()); + assertThat(((DocumentPut)(p.getDocumentOperations().get(0))).getDocument(), sameInstance(inner1)); + assertThat(((DocumentPut)(p.getDocumentOperations().get(1))).getDocument(), sameInstance(inner2)); + assertThat(((DocumentPut)(p.getVariable(cfg.contextFieldName()))).getDocument(), sameInstance(outer)); + assertThat(outer.getFieldValue("innerdocuments"), sameInstance(innerArray)); + assertTrue(innerArray.isEmpty()); + + + JoinerDocumentProcessor joiner = new JoinerDocumentProcessor(cfg, docManCfg); + + joiner.process(p); + + assertThat(p.getDocumentOperations().size(), equalTo(1)); + assertThat(((DocumentPut)p.getDocumentOperations().get(0)).getDocument(), sameInstance(outer)); + assertThat(p.getVariable(cfg.contextFieldName()), nullValue()); + assertThat(outer.getFieldValue("innerdocuments"), sameInstance(innerArray)); + assertThat(innerArray.size(), equalTo(2)); + assertThat(innerArray.get(0), sameInstance(inner1)); + assertThat(innerArray.get(1), sameInstance(inner2)); + } + +} diff --git a/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd b/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd new file mode 100644 index 00000000000..83bc4254fb2 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/util/docindoc.sd @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search docindoc { + document docindoc { + field name type string { header } + field content type string { body } + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg b/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg new file mode 100644 index 00000000000..3347c3127b5 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/util/documentmanager.docindoc.cfg @@ -0,0 +1,42 @@ +enablecompression false +datatype[7] +datatype[0].id -1407012075 +datatype[0].structtype[1] +datatype[0].structtype[0].name "outerdoc.body" +datatype[0].structtype[0].version 0 +datatype[0].structtype[0].field[1] +datatype[0].structtype[0].field[0].datatype -2035324352 +datatype[0].structtype[0].field[0].name "innerdocuments" +datatype[1].id -1686125086 +datatype[1].structtype[1] +datatype[1].structtype[0].name "docindoc.header" +datatype[1].structtype[0].version 0 +datatype[1].structtype[0].field[1] +datatype[1].structtype[0].field[0].datatype 2 +datatype[1].structtype[0].field[0].name "name" +datatype[2].id -2035324352 +datatype[2].arraytype[1] +datatype[2].arraytype[0].datatype 1447635645 +datatype[3].id -2040625920 +datatype[3].structtype[1] +datatype[3].structtype[0].name "outerdoc.header" +datatype[3].structtype[0].version 0 +datatype[4].id 1447635645 +datatype[4].documenttype[1] +datatype[4].documenttype[0].bodystruct 2030224503 +datatype[4].documenttype[0].headerstruct -1686125086 +datatype[4].documenttype[0].name "docindoc" +datatype[4].documenttype[0].version 0 +datatype[5].id 1748635999 +datatype[5].documenttype[1] +datatype[5].documenttype[0].bodystruct -1407012075 +datatype[5].documenttype[0].headerstruct -2040625920 +datatype[5].documenttype[0].name "outerdoc" +datatype[5].documenttype[0].version 0 +datatype[6].id 2030224503 +datatype[6].structtype[1] +datatype[6].structtype[0].name "docindoc.body" +datatype[6].structtype[0].version 0 +datatype[6].structtype[0].field[1] +datatype[6].structtype[0].field[0].datatype 2 +datatype[6].structtype[0].field[0].name "content" diff --git a/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd b/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd new file mode 100644 index 00000000000..dbdf3c32cd5 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/util/outerdoc.sd @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search outerdoc { + document outerdoc { + field innerdocuments type array<docindoc> { body } + } +} diff --git a/docproc/src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg b/docproc/src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg new file mode 100644 index 00000000000..16c26f7a6d8 --- /dev/null +++ b/docproc/src/test/java/com/yahoo/docproc/util/splitter-joiner-document-processor.cfg @@ -0,0 +1,2 @@ +documentTypeName "outerdoc" +arrayFieldName "innerdocuments" diff --git a/docproc/src/test/vespa-configdef/string.def b/docproc/src/test/vespa-configdef/string.def new file mode 100644 index 00000000000..cfc0ecc578c --- /dev/null +++ b/docproc/src/test/vespa-configdef/string.def @@ -0,0 +1,5 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +version=1 + +namespace=config.docproc +stringVal string default="_default_" |