diff options
| author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
| commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
| tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /vespaclient-container-plugin/src/test | |
Publish
Diffstat (limited to 'vespaclient-container-plugin/src/test')
39 files changed, 5846 insertions, 0 deletions
diff --git a/vespaclient-container-plugin/src/test/application/services.xml b/vespaclient-container-plugin/src/test/application/services.xml new file mode 100644 index 00000000000..df178e109c3 --- /dev/null +++ b/vespaclient-container-plugin/src/test/application/services.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<jdisc version="1.0" jetty="true"> + + <handler id="com.yahoo.document.restapi.resource.RestApiWithTestDocumentHandler" bundle="integration-test"> + <binding>http://*/document/v1/*</binding> + </handler> + + <component id="injected" class="com.yahoo.document.restapi.resource.MockedOperationHandler" bundle="integration-test"> + </component> + + + <http> + <!-- This indicates that we want JDisc to allocate a port for us --> + <server id="mainServer" port="0" /> + </http> +</jdisc> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg b/vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg new file mode 100644 index 00000000000..6878ddffb79 --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/documentmanager.cfg @@ -0,0 +1,113 @@ +enablecompression false +datatype[10] +datatype[0].id 1002 +datatype[0].arraytype[1] +datatype[0].arraytype[0].datatype 2 +datatype[0].weightedsettype[0] +datatype[0].structtype[0] +datatype[0].documenttype[0] +datatype[1].id 1000 +datatype[1].arraytype[1] +datatype[1].arraytype[0].datatype 0 +datatype[1].weightedsettype[0] +datatype[1].structtype[0] +datatype[1].documenttype[0] +datatype[2].id 1004 +datatype[2].arraytype[1] +datatype[2].arraytype[0].datatype 4 +datatype[2].weightedsettype[0] +datatype[2].structtype[0] +datatype[2].documenttype[0] +datatype[3].id 1016 +datatype[3].arraytype[1] +datatype[3].arraytype[0].datatype 16 +datatype[3].weightedsettype[0] +datatype[3].structtype[0] +datatype[3].documenttype[0] +datatype[4].id 1001 +datatype[4].arraytype[1] +datatype[4].arraytype[0].datatype 1 +datatype[4].weightedsettype[0] +datatype[4].structtype[0] +datatype[4].documenttype[0] +datatype[5].id 2001 +datatype[5].arraytype[0] +datatype[5].weightedsettype[1] +datatype[5].weightedsettype[0].datatype 0 +datatype[5].weightedsettype[0].createifnonexistant false +datatype[5].weightedsettype[0].removeifzero false +datatype[5].structtype[0] +datatype[5].documenttype[0] +datatype[6].id 2002 +datatype[6].arraytype[0] +datatype[6].weightedsettype[1] +datatype[6].weightedsettype[0].datatype 2 +datatype[6].weightedsettype[0].createifnonexistant false +datatype[6].weightedsettype[0].removeifzero false +datatype[6].structtype[0] +datatype[6].documenttype[0] +datatype[7].id -628990518 +datatype[7].arraytype[0] +datatype[7].weightedsettype[0] +datatype[7].structtype[1] +datatype[7].structtype[0].name news.header +datatype[7].structtype[0].version 0 +datatype[7].structtype[0].field[6] +datatype[7].structtype[0].field[0].name url +datatype[7].structtype[0].field[0].id[0] +datatype[7].structtype[0].field[0].datatype 10 +datatype[7].structtype[0].field[1].name title +datatype[7].structtype[0].field[1].id[0] +datatype[7].structtype[0].field[1].datatype 2 +datatype[7].structtype[0].field[2].name last_downloaded +datatype[7].structtype[0].field[2].id[0] +datatype[7].structtype[0].field[2].datatype 0 +datatype[7].structtype[0].field[3].name value_long +datatype[7].structtype[0].field[3].id[0] +datatype[7].structtype[0].field[3].datatype 4 +datatype[7].structtype[0].field[4].name value_content +datatype[7].structtype[0].field[4].id[0] +datatype[7].structtype[0].field[4].datatype 3 +datatype[7].structtype[0].field[5].name stringarr +datatype[7].structtype[0].field[5].id[0] +datatype[7].structtype[0].field[5].datatype 1002 +datatype[7].documenttype[0] +datatype[8].id 538588767 +datatype[8].arraytype[0] +datatype[8].weightedsettype[0] +datatype[8].structtype[1] +datatype[8].structtype[0].name news.body +datatype[8].structtype[0].version 0 +datatype[8].structtype[0].field[7] +datatype[8].structtype[0].field[0].name intarr +datatype[8].structtype[0].field[0].id[0] +datatype[8].structtype[0].field[0].datatype 1000 +datatype[8].structtype[0].field[1].name longarr +datatype[8].structtype[0].field[1].id[0] +datatype[8].structtype[0].field[1].datatype 1004 +datatype[8].structtype[0].field[2].name bytearr +datatype[8].structtype[0].field[2].id[0] +datatype[8].structtype[0].field[2].datatype 1016 +datatype[8].structtype[0].field[3].name floatarr +datatype[8].structtype[0].field[3].id[0] +datatype[8].structtype[0].field[3].datatype 1001 +datatype[8].structtype[0].field[4].name weightedsetint +datatype[8].structtype[0].field[4].id[0] +datatype[8].structtype[0].field[4].datatype 2001 +datatype[8].structtype[0].field[5].name weightedsetstring +datatype[8].structtype[0].field[5].id[0] +datatype[8].structtype[0].field[5].datatype 2002 +datatype[8].structtype[0].field[6].name content +datatype[8].structtype[0].field[6].id[0] +datatype[8].structtype[0].field[6].datatype 3 +datatype[8].documenttype[0] +datatype[9].id -1048827947 +datatype[9].arraytype[0] +datatype[9].weightedsettype[0] +datatype[9].structtype[0] +datatype[9].documenttype[1] +datatype[9].documenttype[0].name news +datatype[9].documenttype[0].version 0 +datatype[9].documenttype[0].inherits[0] +datatype[9].documenttype[0].headerstruct -628990518 +datatype[9].documenttype[0].bodystruct 538588767 diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test10.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test10.xml new file mode 100644 index 00000000000..52739246b1f --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test10.xml @@ -0,0 +1,91 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<!-- + Document : test10.xml + Created on : July 27, 2007, 11:37 AM + Author : alimf + Description: + this feed contains both documents, updates and removes. +--> + +<vespafeed> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> + + <document documenttype="news" documentid="doc:news:http://news10b"> + <url>testUrl2</url> + </document> + + <update documenttype="news" documentid="doc:news:http://news10c"> + <add field="stringarr"> + <item>addString1</item> + <item>addString2</item> + </add> + <add field="longarr"> + <item>5</item> + </add> + <add field="longarr">6</add> + <add field="weightedsetint"> + <item weight="11">11</item> + <item weight="12">12</item> + </add> + <add field="weightedsetstring"> + <item>add13</item> + </add> + <add field="weightedsetstring">add14</add> + </update> + + <update documenttype="news" documentid="doc:news:http://news10d"> + <assign field="url">assignUrl</assign> + <assign field="value_long">2</assign> + <assign field="stringarr"> + <item>assignString1</item> + <item>assignString2</item> + </assign> + <assign field="intarr"> + <item>3</item> + <item>4</item> + </assign> + <assign field="weightedsetint"> + <item weight="11">11</item> + <item weight="12">12</item> + </assign> + </update> + + <remove documentid="doc:news:http://news10e"/> +</vespafeed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml new file mode 100644 index 00000000000..44762f594e0 --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test10b.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<!-- + Document : test10.xml + Created on : July 27, 2007, 11:37 AM + Author : alimf + Description: + this feed contains both documents, updates and removes. +--> + +<vespafeed> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> + + <document documenttype="news" documentid="doc:news:http://news10b"> + <url>testUrl2</url> + </document> +</vespafeed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml new file mode 100644 index 00000000000..a62df0c645a --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid.xml @@ -0,0 +1,48 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<vespafeed> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> + + <document documenttype="news" documentid="foobar:news:http://news12"> + <url>testUrl2</url> + </document> + + <document documenttype="news" documentid="doc:news:ok"> + <url>testUrl2</url> + </document> +</vespafeed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml new file mode 100755 index 00000000000..434eaa7789c --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_docid_first.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<vespafeed> + <document documenttype="news" documentid="foobar:news:http://news12"> + <url>testUrl2</url> + </document> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> +</vespafeed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml new file mode 100755 index 00000000000..f0b36ea3bc9 --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_bogus_xml.xml @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<vespafeed> + + <document documenttype="news" documentid="doc:news:http://news10a"> + <url>testUrl</url> + <title>testTitle</title> + <last_downloaded>1</last_downloaded> + <value_long>2</value_long> + <value_content>testValueContent</value_content> + <stringarr> + <item>stringarrItem1</item> + <item>stringarrItem2</item> + </stringarr> + <intarr> + <item>3</item> + <item>4</item> + </intarr> + <longarr> + <item>5</item> + <item>6</item> + </longarr> + <bytearr> + <item>7</item> + <item>8</item> + </bytearr> + <floatarr> + <item>9</item> + <item>10</item> + </floatarr> + <weightedsetint> + <item weight="11">11</item> + <item weight="12">12</item> + </weightedsetint> + <weightedsetstring> + <item weight="13">string13</item> + <item weight="14">string14</item> + </weightedsetstring> + </document> + + <document documenttype=news documentid=foobar:news:http://news12 + <url>testUrl2</url> + </document> +</vespafed> diff --git a/vespaclient-container-plugin/src/test/files/feedhandler/test_removes b/vespaclient-container-plugin/src/test/files/feedhandler/test_removes new file mode 100755 index 00000000000..0232970ed4f --- /dev/null +++ b/vespaclient-container-plugin/src/test/files/feedhandler/test_removes @@ -0,0 +1,2 @@ +doc:test:remove1 +doc:test:remove2 diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java new file mode 100644 index 00000000000..87360fcf998 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/OperationHandlerImplTest.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +import com.yahoo.vespaclient.ClusterDef; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + + +public class OperationHandlerImplTest { + + @Test(expected = IllegalArgumentException.class) + public void missingClusterDef() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + OperationHandlerImpl.resolveClusterRoute(Optional.empty(), clusterDef); + } + + @Test(expected = IllegalArgumentException.class) + public void missingClusterDefSpecifiedCluster() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + OperationHandlerImpl.resolveClusterRoute(Optional.of("cluster"), clusterDef); + } + + @Test(expected = RestApiException.class) + public void oneClusterPresentNotMatching() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + clusterDef.add(new ClusterDef("foo", "configId")); + OperationHandlerImpl.resolveClusterRoute(Optional.of("cluster"), clusterDef); + } + + @Test() + public void oneClusterMatching() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + clusterDef.add(new ClusterDef("foo", "configId")); + assertThat(OperationHandlerImpl.resolveClusterRoute(Optional.of("foo"), clusterDef), + is("[Storage:cluster=foo;clusterconfigid=configId]")); + } + + @Test() + public void oneClusterMatchingManyAvailable() throws RestApiException { + List<ClusterDef> clusterDef = new ArrayList<>(); + clusterDef.add(new ClusterDef("foo2", "configId2")); + clusterDef.add(new ClusterDef("foo", "configId")); + clusterDef.add(new ClusterDef("foo3", "configId2")); + assertThat(OperationHandlerImpl.resolveClusterRoute(Optional.of("foo"), clusterDef), + is("[Storage:cluster=foo;clusterconfigid=configId]")); + } + + @Test() + public void checkErrorMessage() throws RestApiException, IOException { + List<ClusterDef> clusterDef = new ArrayList<>(); + clusterDef.add(new ClusterDef("foo2", "configId2")); + clusterDef.add(new ClusterDef("foo", "configId")); + clusterDef.add(new ClusterDef("foo3", "configId2")); + try { + OperationHandlerImpl.resolveClusterRoute(Optional.of("wrong"), clusterDef); + } catch(RestApiException e) { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + e.getResponse().render(stream); + String errorMsg = new String( stream.toByteArray()); + assertThat(errorMsg, is("{\"errors\":[\"Your vespa cluster contains the content clusters foo2 " + + "(configId2), foo (configId), foo3 (configId2), not wrong. Please select a valid vespa cluster.\"]}")); + return; + } + fail("Expected exception"); + } +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java new file mode 100644 index 00000000000..4bb7a264aba --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/RestUriTest.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi; + +import org.apache.http.client.utils.URIBuilder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +public class RestUriTest { + + URI createUri(String path, String query) throws URISyntaxException { + return new URIBuilder() + .addParameter("foo", "bar") + .setHost("host") + .setScheme("http") + .setPort(666) + .setPath(path) + .setCustomQuery(query) + .setFragment("fargment").build(); + } + + @Rule + public ExpectedException thrown= ExpectedException.none(); + + @Test + public void testBasic() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/myid", "query")); + assertThat(restUri.getDocId(), is("myid")); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getNamespace(), is("namespace")); + assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); + assertThat(restUri.generateFullId(), is("id:namespace:doctype::myid")); + } + + @Test + public void encodingSlashes() throws Exception { + // Try with slashes encoded. + final String id = " !\"øæåp/:;&,.:;'1Q"; + String encodedId = URLEncoder.encode(id, StandardCharsets.UTF_8.name()); + RestUri restUri = new RestUri(URI.create("/document/v1/namespace/doctype/docid/" + encodedId)); + assertThat(restUri.getDocId(), is(id)); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getNamespace(), is("namespace")); + assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); + assertThat(restUri.generateFullId(), is("id:namespace:doctype::" + id)); + } + + @Test + public void encodingSlashes2() throws Exception { + // This will decode the slashes. + final String id = " !\"øæåp/:;&,.:;'1Q "; + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/" + id, "query")); + assertThat(restUri.getDocId(), is(id)); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getNamespace(), is("namespace")); + assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); + assertThat(restUri.generateFullId(), is("id:namespace:doctype::" + id)); + } + + + @Test + public void testVisit() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/", "query")); + assertThat(restUri.getDocId(), is("")); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getNamespace(), is("namespace")); + assertThat(restUri.getGroup(), is(Optional.<RestUri.Group>empty())); + assertThat(restUri.generateFullId(), is("id:namespace:doctype::")); + } + + @Test + public void testOneSlashTooMuchWhichIsFine() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/docid/myid:342:23/wrong", "")); + assertThat(restUri.getDocId(), is("myid:342:23/wrong")); + } + + @Test + public void testGroupG() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/group/group/myid", "")); + assertThat(restUri.getDocId(), is("myid")); + assertThat(restUri.getDocumentType(), is("doctype")); + assertThat(restUri.getGroup().get().name, is('g')); + assertThat(restUri.getGroup().get().value, is("group")); + assertThat(restUri.generateFullId(), is("id:namespace:doctype:g=group:myid")); + } + + @Test + public void testGroupN() throws Exception { + RestUri restUri = new RestUri(createUri("/document/v1/namespace/doctype/number/group/myid", "")); + assertThat(restUri.getGroup().get().name, is('n')); + assertThat(restUri.getGroup().get().value, is("group")); + } + + @Test + public void testGroupUnknown() throws Exception { + thrown.expect(RestApiException.class); + new RestUri(createUri("/document/v1/namespace/doctype/Q/myid", "")); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/feed-document1.json diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java new file mode 100644 index 00000000000..ab5b36c74d1 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi.resource; + +import com.yahoo.document.restapi.OperationHandler; +import com.yahoo.document.restapi.Response; +import com.yahoo.document.restapi.RestApiException; +import com.yahoo.document.restapi.RestUri; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; + +import java.util.Optional; + +/** + * Mock that collects info about operation and returns them on second delete. + */ +public class MockedOperationHandler implements OperationHandler { + + StringBuilder log = new StringBuilder(); + int deleteCount = 0; + + @Override + public VisitResult visit(RestUri restUri, String documentSelection, Optional<String> cluster, Optional<String> continuation) throws RestApiException { + return new VisitResult(Optional.of("token"), "List of json docs, cont token " + continuation.map(a->a).orElse("not set") + ", doc selection: '" + + documentSelection + "'"); + } + + @Override + public void put(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException { + log.append("PUT: " + data.getDocument().getId()); + log.append(data.getDocument().getBody().toString()); + } + + @Override + public void update(RestUri restUri, VespaXMLFeedReader.Operation data) throws RestApiException { + log.append("UPDATE: " + data.getDocumentUpdate().getId()); + log.append(data.getDocumentUpdate().getFieldUpdates().toString()); + if (data.getDocumentUpdate().getCreateIfNonExistent()) { + log.append("[CREATE IF NON EXISTENT IS TRUE]"); + } + } + + @Override + public void delete(RestUri restUri, String condition) throws RestApiException { + deleteCount++; + if (deleteCount == 2) { + String theLog = log.toString(); + log = new StringBuilder(); + deleteCount = 0; + throw new RestApiException(Response.createErrorResponse(666, theLog)); + } + log.append("DELETE: " + restUri.generateFullId()); + } + + @Override + public Optional<String> get(RestUri restUri) throws RestApiException { + log.append("GET: " + restUri.generateFullId()); + return Optional.empty(); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java new file mode 100644 index 00000000000..17a36c142ea --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiMaxThreadTest.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi.resource; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.document.restapi.OperationHandler; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; + +public class RestApiMaxThreadTest { + final CountDownLatch latch = new CountDownLatch(1); + final AtomicInteger requestsInFlight = new AtomicInteger(0); + private class RestApiMocked extends RestApi { + + public RestApiMocked() { + super(mock(Executor.class), null, (OperationHandler)null); + } + + @Override + protected HttpResponse handleInternal(HttpRequest request) { + requestsInFlight.incrementAndGet(); + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return null; + } + } + + @Test + public void testCallsAreThrottled() throws InterruptedException { + RestApiMocked restApiMocked = new RestApiMocked(); + // Fire lots of requests. + for (int x = 0; x < 30; x++) { + new Thread(() -> restApiMocked.handle(null)).start(); + } + // Wait for all threads to be used + while (requestsInFlight.get() != 19) { + Thread.sleep(1); + } + // A new request should be blocked. + final HttpResponse response = restApiMocked.handle(null); + assertThat(response.getStatus(), is(429)); + latch.countDown(); + } +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java new file mode 100644 index 00000000000..036dc63ad34 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java @@ -0,0 +1,298 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi.resource; + +import com.yahoo.application.Application; +import com.yahoo.application.Networking; +import com.yahoo.application.container.handler.Request; +import com.yahoo.container.Container; +import com.yahoo.jdisc.http.server.jetty.JettyHttpServer; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Paths; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNot.not; +import static org.hamcrest.core.StringContains.containsString; +import static org.hamcrest.core.StringStartsWith.startsWith; +import static org.junit.Assert.assertThat; + +public class RestApiTest { + Application application; + + @Before + public void setup() throws Exception { + application = Application.fromApplicationPackage(Paths.get("src/test/application"), Networking.enable); + } + + @After + public void tearDown() throws Exception { + application.close(); + } + + String post_test_uri = "/document/v1/namespace/testdocument/docid/c"; + String post_test_doc = "{\n" + + "\"foo\" : \"bar\"," + + "\"fields\": {\n" + + "\"title\": \"This is the title\",\n" + + "\"body\": \"This is the body\"" + + "}" + + "}"; + String post_test_response = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + // Run this test to manually do request against the REST-API with backend mock. + @Ignore + @Test + public void blockingTest() throws Exception { + System.out.println("Running on port " + getFirstListenPort()); + Thread.sleep(Integer.MAX_VALUE); + } + + @Test + public void testbasicPost() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri); + HttpPost httpPost = new HttpPost(request.getUri()); + StringEntity entity = new StringEntity(post_test_doc, ContentType.create("application/json")); + httpPost.setEntity(entity); + String x = doRest(httpPost); + assertThat(x, is(post_test_response)); + } + + String post_test_uri_cond = "/document/v1/namespace/testdocument/docid/c?condition=foo"; + String post_test_doc_cond = "{\n" + + "\"foo\" : \"bar\"," + + "\"fields\": {\n" + + "\"title\": \"This is the title\",\n" + + "\"body\": \"This is the body\"" + + "}" + + "}"; + String post_test_response_cond = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + @Test + public void testConditionalPost() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri_cond); + HttpPost httpPost = new HttpPost(request.getUri()); + StringEntity entity = new StringEntity(post_test_doc_cond, ContentType.create("application/json")); + httpPost.setEntity(entity); + String x = doRest(httpPost); + assertThat(x, is(post_test_response_cond)); + } + + String post_test_empty_response = "{\"errors\":[\"Could not read document, no document?\"]"; + @Test + public void testEmptyPost() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + post_test_uri); + HttpPost httpPost = new HttpPost(request.getUri()); + StringEntity entity = new StringEntity("", ContentType.create("application/json")); + httpPost.setEntity(entity); + String x = doRest(httpPost); + assertThat(x, startsWith(post_test_empty_response)); + } + + String update_test_uri = "/document/v1/namespace/testdocument/docid/c"; + String update_test_doc = "{\n" + + "\t\"fields\": {\n" + + "\"title\": {\n" + + "\"assign\": \"Oh lala\"\n" + + "}\n" + + "}\n" + + "}\n"; + + String update_test_response = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + @Test + public void testbasicUpdate() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_uri); + HttpPut httpPut = new HttpPut(request.getUri()); + StringEntity entity = new StringEntity(update_test_doc, ContentType.create("application/json")); + httpPut.setEntity(entity); + assertThat(doRest(httpPut), is(update_test_response)); + assertThat(getLog(), not(containsString("CREATE IF NON EXISTING IS TRUE"))); + } + + @Test + public void testbasicUpdateCreateTrue() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_uri + "?create=true"); + HttpPut httpPut = new HttpPut(request.getUri()); + StringEntity entity = new StringEntity(update_test_doc, ContentType.create("application/json")); + httpPut.setEntity(entity); + assertThat(doRest(httpPut), is(update_test_response)); + assertThat(getLog(), containsString("CREATE IF NON EXISTENT IS TRUE")); + } + + String update_test_create_if_non_existient_uri = "/document/v1/namespace/testdocument/docid/c"; + String update_test_create_if_non_existient_doc = "{\n" + + "\"create\":true," + + "\t\"fields\": {\n" + + "\"title\": {\n" + + "\"assign\": \"Oh lala\"\n" + + "}\n" + + "}\n" + + "}\n"; + + String update_test_create_if_non_existing_response = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + @Test + public void testCreateIfNonExistingUpdateInDocTrue() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_create_if_non_existient_uri); + HttpPut httpPut = new HttpPut(request.getUri()); + StringEntity entity = new StringEntity(update_test_create_if_non_existient_doc, ContentType.create("application/json")); + httpPut.setEntity(entity); + assertThat(doRest(httpPut), is(update_test_create_if_non_existing_response)); + assertThat(getLog(), containsString("CREATE IF NON EXISTENT IS TRUE")); + + } + + @Test + public void testCreateIfNonExistingUpdateInDocTrueButQueryParamsFalse() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + update_test_create_if_non_existient_uri + "?create=false"); + HttpPut httpPut = new HttpPut(request.getUri()); + StringEntity entity = new StringEntity(update_test_create_if_non_existient_doc, ContentType.create("application/json")); + httpPut.setEntity(entity); + assertThat(doRest(httpPut), is(update_test_create_if_non_existing_response)); + assertThat(getLog(), not(containsString("CREATE IF NON EXISTENT IS TRUE"))); + + } + + // Get logs through some hackish fetch method. Logs is something the mocked backend write. + String getLog() throws IOException { + // The mocked backend will throw a runtime exception wtih a log if delete is called three times.. + Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri); + HttpDelete delete = new HttpDelete(request.getUri()); + doRest(delete); + return doRest(delete); + } + + + String remove_test_uri = "/document/v1/namespace/testdocument/docid/c"; + String remove_test_response = "{\"id\":\"id:namespace:testdocument::c\"," + + "\"pathId\":\"/document/v1/namespace/testdocument/docid/c\"}"; + + @Test + public void testbasicRemove() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + remove_test_uri); + HttpDelete delete = new HttpDelete(request.getUri()); + assertThat(doRest(delete), is(remove_test_response)); + } + + String get_test_uri = "/document/v1/namespace/document-type/docid/c"; + String get_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/c\""; + String get_response_part2 = "\"id\":\"id:namespace:document-type::c\""; + + + @Test + public void testbasicGet() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + get_test_uri); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(get_response_part1)); + assertThat(rest, containsString(get_response_part2)); + } + + String id_test_uri = "/document/v1/namespace/document-type/docid/f/u/n/n/y/!"; + String id_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/f/u/n/n/y/!\""; + String id_response_part2 = "\"id\":\"id:namespace:document-type::f/u/n/n/y/!\""; + + @Test + public void testSlashesInId() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + id_test_uri); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(id_response_part1)); + assertThat(rest, containsString(id_response_part2)); + } + + + String get_enc_id = "!\":æøå@/& Q1+"; + // Space encoded as %20, not encoding ! + String get_enc_id_encoded_v1 = "!%22%3A%C3%A6%C3%B8%C3%A5%40%2F%26%20Q1%2B"; + // Space encoded as + + String get_enc_id_encoded_v2 = "%21%22%3A%C3%A6%C3%B8%C3%A5%40%2F%26+Q1%2B"; + String get_enc_test_uri_v1 = "/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v1; + String get_enc_test_uri_v2 = "/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v2; + String get_enc_response_part1 = "\"pathId\":\"/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v1 + "\""; + String get_enc_response_part1_v2 = "\"pathId\":\"/document/v1/namespace/document-type/docid/" + get_enc_id_encoded_v2 + "\""; + + // JSON encode " as \" + String get_enc_response_part2 = "\"id\":\"id:namespace:document-type::" + get_enc_id.replace("\"", "\\\"") + "\""; + + + @Test + public void testbasicEncodingV1() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + get_enc_test_uri_v1); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(get_enc_response_part1)); + assertThat(rest, containsString(get_enc_response_part2)); + } + + @Test + public void testbasicEncodingV2() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + get_enc_test_uri_v2); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(get_enc_response_part1_v2)); + assertThat(rest, containsString(get_enc_response_part2)); + } + + String visit_test_uri = "/document/v1/namespace/document-type/docid/?continuation=abc"; + String visit_response_part1 = "\"documents\":[List of json docs, cont token abc, doc selection: '']"; + String visit_response_part2 = "\"continuation\":\"token\""; + String visit_response_part3 = "\"pathId\":\"/document/v1/namespace/document-type/docid/\""; + + + @Test + public void testbasicVisit() throws Exception { + Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_uri); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(visit_response_part1)); + assertThat(rest, containsString(visit_response_part2)); + assertThat(rest, containsString(visit_response_part3)); + } + + String visit_test_bad_uri = "/document/v1/namespace/document-type/group/abc?continuation=abc"; + String visit_test_bad_response = "Visiting does not support setting value for group/value,"; + + + @Test + public void testBadVisit() throws Exception { + final Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_bad_uri); + HttpGet get = new HttpGet(request.getUri()); + final String rest = doRest(get); + assertThat(rest, containsString(visit_test_bad_response)); + } + + private String doRest(HttpRequestBase request) throws IOException { + HttpClient client = HttpClientBuilder.create().build(); + HttpResponse response = client.execute(request); + HttpEntity entity = response.getEntity(); + return EntityUtils.toString(entity); + } + + private String getFirstListenPort() { + JettyHttpServer serverProvider = + (JettyHttpServer) Container.get().getServerProviderRegistry().allComponents().get(0); + return Integer.toString(serverProvider.getListenPort()); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java new file mode 100644 index 00000000000..cfb120d9891 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiWithTestDocumentHandler.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.restapi.resource; + +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.restapi.OperationHandler; + +import java.util.concurrent.Executor; + +/** + * For setting up RestApi with a simple document type manager. + * + * @author dybdahl + */ +public class RestApiWithTestDocumentHandler extends RestApi{ + + private DocumentTypeManager docTypeManager = new DocumentTypeManager(); + + public RestApiWithTestDocumentHandler( + Executor executor, + AccessLog accessLog, + OperationHandler operationHandler) { + super(executor, accessLog, operationHandler); + + DocumentType documentType = new DocumentType("testdocument"); + + documentType.addField("title", DataType.STRING); + documentType.addField("body", DataType.STRING); + docTypeManager.registerDocumentType(documentType); + + setDocTypeManagerForTests(docTypeManager); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/externalfeeding/server/.gitignore diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java new file mode 100644 index 00000000000..6d0ee59c68d --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/FeedHandlerTest.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.HeaderFields; +import com.yahoo.jdisc.Metric; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespa.http.server.FeedHandler; +import com.yahoo.vespa.http.server.Feeder; +import org.junit.Test; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit test for FeedHandler class. + * + * @author dybdahl + */ +public class FeedHandlerTest { + + /** + * This class extends FeedHandler and allows to create a custom Feeder. + */ + static class TestFeedHandler extends FeedHandler { + private final CountDownLatch countDownLatch = new CountDownLatch(1); + + public TestFeedHandler() throws Exception { + super(Executors.newCachedThreadPool(), null, null, mock(Metric.class), mock(AccessLog.class), null); + } + + /** + * Builds a feeder that blocks until countDownLatch is stepped down. + */ + @Override + protected Feeder createFeeder( + com.yahoo.container.jdisc.HttpRequest request, + InputStream requestInputStream, + final BlockingQueue<OperationStatus> operations, + String clientId, + boolean sessionIdWasGeneratedJustNow, + int protocolVersion) throws Exception { + Feeder feeder = mock(Feeder.class); + doAnswer(invocation -> { + try { + countDownLatch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return null; + }).when(feeder).waitForRequestReceived(); + return feeder; + } + } + + /** + * nginx require that a post is finished before the server ack with a response. This behaviour is verified + * in this test + */ + @Test + public void testResponseIsSentAfterWaitForRequestReceivedReturns() throws Exception { + HttpRequest request = mock(HttpRequest.class); + + // Create a request with valid version. + com.yahoo.jdisc.http.HttpRequest jdiscRequest = mock(com.yahoo.jdisc.http.HttpRequest.class); + HeaderFields headerFields = mock(HeaderFields.class); + List<String> version = new ArrayList<>(); + version.add("2"); + when(headerFields.get(Headers.VERSION)).thenReturn(version); + when(jdiscRequest.headers()).thenReturn(headerFields); + when(request.getJDiscRequest()).thenReturn(jdiscRequest); + + TestFeedHandler feedHandler = new TestFeedHandler(); + // After a short period, make the feed finish. + new Thread(() -> { + try { + Thread.sleep(50); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + feedHandler.countDownLatch.countDown(); + }).start(); + // This should not return before countdown latch is stepped down. + feedHandler.handle(request); + // This should always returns after the countDownLatch has become zero. This might cause false positive, + // but not false negatives. This is fine. + assertThat(feedHandler.countDownLatch.getCount(), is(0L)); + + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java new file mode 100755 index 00000000000..646bcb805f6 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/VespaFeedHandlerTestCase.java @@ -0,0 +1,1015 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.docproc.CallStack; +import com.yahoo.jdisc.HeaderFields; +import com.yahoo.messagebus.*; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.container.Container; +import com.yahoo.docproc.*; +import com.yahoo.docproc.jdisc.DocumentProcessingHandler; +import com.yahoo.docproc.jdisc.DocumentProcessingHandlerParameters; +import com.yahoo.document.*; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.feedapi.DummySessionFactory; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaclient.ClusterList; +import com.yahoo.vespaclient.config.FeederConfig; +import org.junit.After; +import org.junit.Test; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.logging.Logger; +import java.util.zip.GZIPOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class VespaFeedHandlerTestCase { + + private VespaFeedHandler feedHandler; + private VespaFeedHandlerRemove removeHandler; + private VespaFeedHandlerStatus statusHandler; + private VespaFeedHandlerRemoveLocation removeLocationHandler; + private FeedContext context; + + private DummySessionFactory factory; + private final String xmlFilesPath = "src/test/files/feedhandler/"; + + public void setup(com.yahoo.messagebus.Error e, LoadTypeConfig loadTypeCfg, + boolean autoReply, + DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { + DocumentTypeManager docMan = new DocumentTypeManager(); + DocumentTypeManagerConfigurer.configure(docMan, "file:" + xmlFilesPath + "documentmanager.cfg"); + + if (autoReply) { + if (autoReplyFactory != null) { + factory = DummySessionFactory.createWithAutoReplyFactory(autoReplyFactory); + } else { + factory = DummySessionFactory.createWithErrorAutoReply(e); + } + } else { + factory = DummySessionFactory.createDefault(); + } + + context = new FeedContext(new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder()), loadTypeCfg), factory, docMan, new ClusterList(), new NullFeedMetric()); + + Executor threadPool = Executors.newCachedThreadPool(); + feedHandler = new VespaFeedHandler(context, threadPool); + removeHandler = new VespaFeedHandlerRemove(context, threadPool); + statusHandler = new VespaFeedHandlerStatus(context, false, false, threadPool); + removeLocationHandler = new VespaFeedHandlerRemoveLocation(context, threadPool); + + CallStack dpCallstack = new CallStack("bar"); + dpCallstack.addLast(new TestDocProc()); + dpCallstack.addLast(new TestLaterDocProc()); + + DocprocService myservice = new DocprocService("bar"); + myservice.setCallStack(dpCallstack); + myservice.setInService(true); + + ComponentRegistry<DocprocService> registry = new ComponentRegistry<DocprocService>(); + registry.register(new ComponentId(myservice.getName()), myservice); + + DocumentProcessingHandler handler = new DocumentProcessingHandler(registry, + new ComponentRegistry<>(), new ComponentRegistry<>(), + new DocumentProcessingHandlerParameters()); + + Container container = Container.get(); + ComponentRegistry<RequestHandler> requestHandlerComponentRegistry = new ComponentRegistry<>(); + requestHandlerComponentRegistry.register(new ComponentId(DocumentProcessingHandler.class.getName()), handler); + container.setRequestHandlerRegistry(requestHandlerComponentRegistry); + } + + public void setup(com.yahoo.messagebus.Error e) throws Exception { + setup(e, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, null); + } + + public void setupWithReplyFactory(DummySessionFactory.ReplyFactory autoReplyFactory) throws Exception { + setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), true, autoReplyFactory); + } + + public void setup() throws Exception { + setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder()), false, null); + } + + @After + public void resetContainer() { + Container.resetInstance(); + } + + + @Test + public void testLoadTypes() throws Exception { + List<LoadTypeConfig.Type.Builder> typeBuilder = new ArrayList<>(); + typeBuilder.add(new LoadTypeConfig.Type.Builder().id(1234).name("foo").priority("VERY_LOW")); + typeBuilder.add(new LoadTypeConfig.Type.Builder().id(4567).name("bar").priority("NORMAL_3")); + + setup(null, new LoadTypeConfig(new LoadTypeConfig.Builder().type(typeBuilder)), true, null); + + { + Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme&loadtype=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:removeme", d.toString()); + assertEquals(new LoadType(1234, "foo", DocumentProtocol.Priority.VERY_LOW), ((DocumentMessage)m).getLoadType()); + assertEquals(DocumentProtocol.Priority.VERY_LOW, ((DocumentMessage)m).getPriority()); + } + } + + @Test + public void testPostXML() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + assertTrue(res.output.contains("count=\"2\"")); + assertTrue(res.error == null); + } + + @Test + public void testPostXMLAsync() throws Exception { + setup(); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?asynchronous=true"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + // Should not have metrics at this point. + assertTrue(!res.output.contains("count=\"2\"")); + assertTrue(res.error == null); + } + + + @Test + public void testPostGZIPedXML() throws Exception { + setup(null); + Result res = testFeedGZIP(xmlFilesPath + "test10b.xml", "feed?"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + assertTrue(res.error == null); + } + + @Test + public void testDocProc() throws Exception { + setup(null); + + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); + + assertEquals("doc:news:http://news10a", d.getId().toString()); + assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + Document d = ((PutDocumentMessage)m).getDocumentPut().getDocument(); + + assertEquals("doc:news:http://news10b", d.getId().toString()); + assertEquals(new IntegerFieldValue(1234), d.getFieldValue("last_downloaded")); + } + } + + @Test + public void testPostXMLVariousTypes() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test10.xml", "feed?"); + + assertEquals(5, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + { + Message m = res.messages.get(2); + assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); + DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); + assertEquals("doc:news:http://news10c", d.toString()); + } + { + Message m = res.messages.get(3); + assertEquals(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, m.getType()); + DocumentId d = ((UpdateDocumentMessage)m).getDocumentUpdate().getId(); + assertEquals("doc:news:http://news10d", d.toString()); + } + { + Message m = res.messages.get(4); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:news:http://news10e", d.toString()); + } + + String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); + + assertEquals("<result>\n" + + "\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <latency count=\"5\"/>\n" + + " <count count=\"5\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"1\"/>\n" + + " <count count=\"1\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + "\n" + + "</result>\n", val); + } + + @Test + public void testStatusPage() throws Exception { + setup(null); + + testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); + testFeed(xmlFilesPath + "test10.xml", "feed?"); + testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); + testFeed(xmlFilesPath + "test_removes", "remove?"); + + assertEquals(2, factory.sessionsCreated()); + Result res = testRequest(HttpRequest.createTestRequest("feedstatus?", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + + String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); + val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); + + assertEquals("<status>\n" + + "\n" + + " <snapshot name=\"Total metrics from start until current time\" from=\"0\" to=\"0\" period=\"0\">\n" + + " <routes>\n" + + " <route name=\"total\">\n" + + " <total>\n" + + " <latency count=\"14\"/>\n" + + " <count count=\"14\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"6\"/>\n" + + " <count count=\"6\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <latency count=\"9\"/>\n" + + " <count count=\"9\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"3\"/>\n" + + " <count count=\"3\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " <route name=\"storage\">\n" + + " <total>\n" + + " <latency count=\"5\"/>\n" + + " <count count=\"5\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"1\"/>\n" + + " <count count=\"1\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " </routes>\n" + + " </snapshot>\n" + + "\n" + + "</status>\n", val); + } + + @Test + public void testStatusPage2() throws Exception { + setup(null); + + testFeed(xmlFilesPath + "test10b.xml", "feed?docprocchain=bar"); + testFeed(xmlFilesPath + "test10.xml", "feed?"); + testFeed(xmlFilesPath + "test10.xml", "feed?route=storage"); + testFeed(xmlFilesPath + "test_removes", "remove?"); + + assertEquals(2, factory.sessionsCreated()); + Result res = testRequest(HttpRequest.createTestRequest("feed?status", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + + String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); + val = val.replaceAll("to=\"[0-9]*\"", "to=\"0\""); + + assertEquals("<status>\n" + + "\n" + + " <routes>\n" + + " <route name=\"total\" description=\"Messages sent to all routes\">\n" + + " <total description=\"All kinds of messages sent to the given route\">\n" + + " <latency count=\"14\"/>\n" + + " <count count=\"14\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"6\"/>\n" + + " <count count=\"6\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " <route name=\"default\" description=\"Messages sent to the named route\">\n" + + " <total description=\"All kinds of messages sent to the given route\">\n" + + " <latency count=\"9\"/>\n" + + " <count count=\"9\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"4\"/>\n" + + " <count count=\"4\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"3\"/>\n" + + " <count count=\"3\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " <route name=\"storage\" description=\"Messages sent to the named route\">\n" + + " <total description=\"All kinds of messages sent to the given route\">\n" + + " <latency count=\"5\"/>\n" + + " <count count=\"5\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </putdocument>\n" + + " <updatedocument>\n" + + " <latency count=\"2\"/>\n" + + " <count count=\"2\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </updatedocument>\n" + + " <removedocument>\n" + + " <latency count=\"1\"/>\n" + + " <count count=\"1\"/>\n" + + " <ignored count=\"0\"/>\n" + + " </removedocument>\n" + + " </route>\n" + + " </routes>\n" + + "\n" + + "</status>\n", val); + } + + @Test + public void testMetricForIgnoredDocumentsIsIncreased() throws Exception { + DummySessionFactory.ReplyFactory replyFactory = new DummySessionFactory.ReplyFactory() { + @Override + public Reply createReply(Message m) { + return new DocumentIgnoredReply(); + } + }; + setupWithReplyFactory(replyFactory); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); + assertEquals(2, res.messages.size()); + + String val = res.output.replaceAll("<([a-z]+).*count=\"([0-9]+)\".*/", "<$1 count=\"$2\"/"); + + assertEquals("<result>\n" + + "\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <ignored count=\"2\"/>\n" + + " </total>\n" + + " <putdocument>\n" + + " <ignored count=\"2\"/>\n" + + " </putdocument>\n" + + " </route>\n" + + "\n" + + "</result>\n", val); + } + + @Test + public void testPostXMLWithMBusFailureAllowed() throws Exception { + setup(new com.yahoo.messagebus.Error(DocumentProtocol.ERROR_BUCKET_DELETED, "Hello world in <document>")); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?abortonfeederror=false"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10b", d.toString()); + } + + String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); + System.out.println(val); + + assertEquals("<result>\n" + + "\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <errors>\n" + + " <error name=\"total\" count=\"2\"/>\n" + + " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + + " </errors>\n" + + " </total>\n" + + " <putdocument>\n" + + " <errors>\n" + + " <error name=\"total\" count=\"2\"/>\n" + + " <error name=\"BUCKET_DELETED\" count=\"2\"/>\n" + + " </errors>\n" + + " </putdocument>\n" + + " </route>\n\n" + + " <errors count=\"2\">\n" + + " <error message=\"PUT[doc:news:http://news10a] [BUCKET_DELETED] Hello world in <document>\"/>\n" + + " <error message=\"PUT[doc:news:http://news10b] [BUCKET_DELETED] Hello world in <document>\"/>\n" + + " </errors>\n" + + "\n" + + "</result>\n", val); + + assertTrue(res.error != null); + assertTrue(res.errorCount > 0); + } + + @Test + public void testPostXMLWithMBusFailure() throws Exception { + setup(new com.yahoo.messagebus.Error(32, "Hello world")); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?"); + + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + + String val = res.output.replaceAll("average=\"[0-9]*\" last=\"[0-9]*\" min=\"[0-9]*\" max=\"[0-9]*\" ", ""); + assertEquals("<result>\n" + + "\n" + + " <route name=\"default\">\n" + + " <total>\n" + + " <errors>\n" + + " <error name=\"total\" count=\"1\"/>\n" + + " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + + " </errors>\n" + + " </total>\n" + + " <putdocument>\n" + + " <errors>\n" + + " <error name=\"total\" count=\"1\"/>\n" + + " <error name=\"UNKNOWN(32)\" count=\"1\"/>\n" + + " </errors>\n" + + " </putdocument>\n" + + " </route>\n\n" + + " <errors count=\"1\">\n" + + " <error message=\"PUT[doc:news:http://news10a] [UNKNOWN(32)] Hello world\"/>\n" + + " </errors>\n" + + "\n" + + "</result>\n", val); + + assertTrue(res.error != null); + assertTrue(res.errorCount > 0); + } + + @Test + public void testPostXMLWithIllegalDocId() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?"); + + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + } + + @Test + public void testPostXMLWithIllegalDocIdAllowFailure() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_docid.xml", "feed?abortondocumenterror=false"); + + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:ok", d.toString()); + } + } + + @Test + public void testPostUnparseableXML() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_xml.xml", "feed?"); + + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + } + + @Test + public void testOverrides() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2.222&route=storage&priority=HIGH_2"); + + assertEquals(2, res.messages.size()); + + for (Message m : res.messages) { + assertEquals(2222, m.getTimeRemaining()); + assertEquals(Route.parse("storage"), m.getRoute()); + assertEquals(DocumentProtocol.Priority.HIGH_2, ((DocumentMessage)m).getPriority()); + } + } + + @Test + public void testBogusPriority() throws Exception { + try { + setup(null); + Result res = testFeed(xmlFilesPath + "test10b.xml", "feed?timeout=2222&route=storage&priority=HIPSTER_DOOFUS"); + assertTrue(false); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void testPostXMLWithIllegalDocIdFirst() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?"); + + assertEquals(0, res.messages.size()); + } + + @Test + public void testPostXMLWithIllegalDocIdFirstNoAbort() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_bogus_docid_first.xml", "feed?abortondocumenterror=false"); + + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_PUTDOCUMENT, m.getType()); + DocumentId d = ((PutDocumentMessage)m).getDocumentPut().getDocument().getId(); + assertEquals("doc:news:http://news10a", d.toString()); + } + } + + @Test + public void testSimpleRemove() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:removeme", d.toString()); + } + } + + @Test + public void testRemoveUser() throws Exception { + setup(null); + + context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); + String selection = ((RemoveLocationMessage)m).getDocumentSelection(); + assertEquals("storage", m.getRoute().toString()); + assertEquals("id.user=1234", selection); + } + } + + @Test + public void testRemoveGroup() throws Exception { + setup(null); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); + String selection = ((RemoveLocationMessage)m).getDocumentSelection(); + assertEquals("storage", m.getRoute().toString()); + assertEquals("id.group=\"foo\"", selection); + } + } + + @Test + public void testRemoveBadSyntax() throws Exception { + setup(null); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo&user=12345", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(0, res.messages.size()); + assertTrue(res.error.toString().contains("Exactly one of")); + } + + @Test + public void testRemoveGroupMultipleClusters() throws Exception { + setup(null); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage1", "storage/cluster.storage1")); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage2", "storage/cluster.storage2")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(0, res.messages.size()); + assertTrue(res.error.toString().contains("More than one")); + } + + @Test + public void testRemoveGroupNoClusters() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?group=foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(0, res.messages.size()); + assertTrue(res.error.toString().contains("No storage clusters")); + } + + @Test + public void testRemoveSelection() throws Exception { + setup(null); + context.getClusterList().getStorageClusters().add(new ClusterDef("storage", "storage/cluster.storage")); + Result res = testRequest(HttpRequest.createTestRequest("removelocation?selection=id.user=1234", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVELOCATION, m.getType()); + String selection = ((RemoveLocationMessage)m).getDocumentSelection(); + assertEquals("id.user=1234", selection); + } + } + + @Test + public void testSimpleRemoveIndex() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(1, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:removeme", d.toString()); + } + } + + @Test + public void testPostRemove() throws Exception { + setup(null); + Result res = testFeed(xmlFilesPath + "test_removes", "remove?"); + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:remove1", d.toString()); + } + + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:remove2", d.toString()); + } + } + + @Test + public void testRemoveBogusId() throws Exception { + try { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id=unknowndoc:test:removeme", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertTrue(false); + } catch (Exception e) { + } + } + + @Test + public void testMultiRemove() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=doc:test:removeme&id[1]=doc:test:remove2&id[2]=doc:test:remove3", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(3, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:removeme", d.toString()); + } + + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:remove2", d.toString()); + } + + { + Message m = res.messages.get(2); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + DocumentId d = ((RemoveDocumentMessage)m).getDocumentId(); + assertEquals("doc:test:remove3", d.toString()); + } + } + + @Test + public void testMultiRemoveSameDoc() throws Exception { + setup(null); + Result res = testRequest(HttpRequest.createTestRequest("remove?id[0]=userdoc:footype:1234:foo&id[1]=userdoc:footype:1234:foo", com.yahoo.jdisc.http.HttpRequest.Method.PUT)); + assertEquals(2, res.messages.size()); + + { + Message m = res.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + } + + { + Message m = res.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, m.getType()); + } + } + + @Test + public void testFeedHandlerStatusCreation() throws Exception { + VespaFeedHandlerStatus status = new VespaFeedHandlerStatus( + new FeedContext(new MessagePropertyProcessor( + new FeederConfig(new FeederConfig.Builder()), + new LoadTypeConfig(new LoadTypeConfig.Builder())), + factory, null, new ClusterList(), new NullFeedMetric()), + true, true, + Executors.newCachedThreadPool()); + } + + private class TestDocProc extends DocumentProcessor { + @Override + public Progress process(Processing processing) { + for (DocumentOperation op : processing.getDocumentOperations()) { + if (op instanceof DocumentPut) { + Document document = ((DocumentPut)op).getDocument(); + document.setFieldValue("last_downloaded", new IntegerFieldValue(1234)); + } + } + return Progress.DONE; + } + } + + private class TestLaterDocProc extends DocumentProcessor { + private final Logger log = Logger.getLogger(TestLaterDocProc.class.getName()); + + private int counter = 0; + @Override + public Progress process(Processing processing) { + synchronized (this) { + counter++; + if (counter % 2 == 1) { + log.info("Returning LATER."); + return Progress.LATER; + } + log.info("Returning DONE."); + return Progress.DONE; + } + } + } + + private Result testRequest(HttpRequest req) throws Exception { + HttpResponse response = null; + String feedPrefix = "feed"; + String removePrefix = "remove"; + String feedStatusPrefix = "feedstatus"; + String removeLocationPrefix = "removelocation"; + + if (req.getUri().getPath().startsWith(feedPrefix)) { + response = feedHandler.handle(req); + } + if (req.getUri().getPath().startsWith(removePrefix)) { + response = removeHandler.handle(req); + } + if (req.getUri().getPath().startsWith(feedStatusPrefix)) { + response = statusHandler.handle(req); + } + if (req.getUri().getPath().startsWith(removeLocationPrefix)) { + response = removeLocationHandler.handle(req); + } + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + response.render(output); + + Result res = new Result(); + res.messages = factory.messages; + res.output = new String(output.toByteArray()); + + if (response instanceof FeedResponse) { + FeedResponse feedResponse = (FeedResponse)response; + res.error = feedResponse.getErrorMessageList().isEmpty() ? null : feedResponse.getErrorMessageList().get(0); + res.errorCount = feedResponse.getErrorMessageList().size(); + assertTrue(feedResponse.isSuccess() == (res.errorCount == 0)); + } + return res; + } + + private Result testFeed(String xmlFile, String request) throws Exception { + return testRequest(new FileRequest(new File(xmlFile), request).toRequest()); + } + + private Result testFeedGZIP(String xmlFile, String request) throws Exception { + return testRequest(new FileRequest(new File(xmlFile), request, true).toRequest()); + } + + private class FileRequest { + + private final String req; + private final File f; + private boolean gzip = false; + + FileRequest(File f, String req) { + this.req = req; + this.f = f; + } + + FileRequest(File f, String req, boolean gzip) { + this.f = f; + this.req = req; + this.gzip = gzip; + } + + public InputStream getData() { + try { + InputStream fileStream = new FileInputStream(f); + if (gzip) { + // Not exactly pretty, but in lack of an elegant way of transcoding + ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); + GZIPOutputStream compressed = new GZIPOutputStream(rawOut); + byte[] buffer = new byte[1024]; + int read = -1; + while (true) { + read = fileStream.read(buffer); + if (read == -1) break; + compressed.write(buffer, 0, read); + } + compressed.finish(); + compressed.flush(); + rawOut.flush(); + return new ByteArrayInputStream(rawOut.toByteArray()); + } + return fileStream; + } catch (Exception e) { + return null; + } + } + + public void addHeaders(HeaderFields headers) { + headers.add("Content-Type", "image/jpeg"); + if (gzip) + headers.add("Content-Encoding", "gzip"); + } + + public HttpRequest toRequest() { + HttpRequest request = HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); + addHeaders(request.getJDiscRequest().headers()); + return request; + } + + } + + private class Result { + private List<Message> messages; + private String output; + private com.yahoo.processing.request.ErrorMessage error; + private int errorCount; + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java new file mode 100644 index 00000000000..708a9ed7a6b --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java @@ -0,0 +1,134 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.feedhandler.v3; + +import com.google.common.base.Splitter; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.http.client.config.FeedParams; +import com.yahoo.vespa.http.client.core.ErrorCode; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespa.http.server.ReplyContext; +import com.yahoo.vespa.http.server.FeedHandlerV3; +import org.junit.Test; +import com.yahoo.container.jdisc.HttpRequest; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import com.yahoo.messagebus.Result; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +public class FeedTesterV3 { + + @Test + public void feedOneDocument() throws Exception { + final FeedHandlerV3 feedHandlerV3 = setupFeederHandler(); + HttpResponse httpResponse = feedHandlerV3.handle(createRequest(1)); + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + httpResponse.render(outStream); + assertThat(httpResponse.getContentType(), is("text/plain")); + assertThat(Utf8.toString(outStream.toByteArray()), is("1230 OK message trace\n")); + + } + + @Test + public void feedManyDocument() throws Exception { + final FeedHandlerV3 feedHandlerV3 = setupFeederHandler(); + HttpResponse httpResponse = feedHandlerV3.handle(createRequest(100)); + ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + httpResponse.render(outStream); + assertThat(httpResponse.getContentType(), is("text/plain")); + String result = Utf8.toString(outStream.toByteArray()); + assertThat(Splitter.on("\n").splitToList(result).size(), is(101)); + } + + DocumentTypeManager createDoctypeManager() { + DocumentTypeManager docTypeManager = new DocumentTypeManager(); + DocumentType documentType = new DocumentType("testdocument"); + documentType.addField("title", DataType.STRING); + documentType.addField("body", DataType.STRING); + docTypeManager.registerDocumentType(documentType); + return docTypeManager; + } + + HttpRequest createRequest(int numberOfDocs) { + String clientId = "client123"; + StringBuilder wireData = new StringBuilder(); + for (int x = 0; x < numberOfDocs; x++) { + String docData = "[{\"put\": \"id:testdocument:testdocument::c\", \"fields\": { \"title\": \"fooKey\", \"body\": \"value\"}}]"; + String operationId = "123" + x; + wireData.append(operationId + " " + Integer.toHexString(docData.length()) + "\n" + docData); + } + InputStream inputStream = new ByteArrayInputStream(wireData.toString().getBytes()); + HttpRequest request = HttpRequest.createTestRequest( + "http://dummyhostname:19020/reserved-for-internal-use/feedapi", + com.yahoo.jdisc.http.HttpRequest.Method.POST, + inputStream); + request.getJDiscRequest().headers().add(Headers.VERSION, "3"); + request.getJDiscRequest().headers().add(Headers.DATA_FORMAT, FeedParams.DataFormat.JSON_UTF8.name()); + request.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + request.getJDiscRequest().headers().add(Headers.CLIENT_ID, clientId); + request.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + request.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + request.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + return request; + } + + FeedHandlerV3 setupFeederHandler() throws Exception { + Executor threadPool = Executors.newCachedThreadPool(); + DocumentmanagerConfig docMan = new DocumentmanagerConfig(new DocumentmanagerConfig.Builder().enablecompression(true)); + FeedHandlerV3 feedHandlerV3 = new FeedHandlerV3( + threadPool, docMan, null /* session cache */ , new NullFeedMetric(), AccessLog.voidAccessLog(), null) { + @Override + protected ReferencedResource<SharedSourceSession> retainSource( + SessionCache sessionCache, SourceSessionParams sessionParams) { + SharedSourceSession sharedSourceSession = mock(SharedSourceSession.class); + + try { + Mockito.stub(sharedSourceSession.sendMessageBlocking(anyObject())).toAnswer((Answer) invocation -> { + Object[] args = invocation.getArguments(); + PutDocumentMessage putDocumentMessage = (PutDocumentMessage) args[0]; + ReplyContext replyContext = (ReplyContext)putDocumentMessage.getContext(); + replyContext.feedReplies.add(new OperationStatus("message", replyContext.docId, ErrorCode.OK, "trace")); + Result result = mock(Result.class); + when(result.isAccepted()).thenReturn(true); + return result; + }); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + Result result = mock(Result.class); + when(result.isAccepted()).thenReturn(true); + ReferencedResource<SharedSourceSession> refSharedSessopn = + new ReferencedResource<>(sharedSourceSession, () -> {}); + return refSharedSessopn; + } + }; + feedHandlerV3.injectDocumentManangerForTests(createDoctypeManager()); + return feedHandlerV3; + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java new file mode 100644 index 00000000000..8b991909cd8 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ContinuationHitTest.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.document.BucketId; +import com.yahoo.documentapi.ProgressToken; +import com.yahoo.documentapi.VisitorIterator; +import org.junit.Test; + +import java.util.Set; +import java.util.TreeSet; + +import static org.junit.Assert.*; + +public class ContinuationHitTest { + + private static final String SINGLE_BUCKET_URL_SAFE_BASE64 + = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAHqNFZ4mrz-_wAAAAAAAAAA"; + private static final String MULTI_BUCKET_URL_SAFE_BASE64 + = "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz--gAAAAAAAAAA6" + + "jRWeJq8_vsAAAAAAAAAAOo0VniavP7_AAAAAAAAAAA="; + + @Test + public void continuationTokensAreUrlSafeBase64Encoded() throws Exception { + ContinuationHit hit = new ContinuationHit(createSingleBucketProgress()); + // We want -_ instead of +/ + assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, hit.getValue()); + } + + @Test + public void continuationTokensAreNotBrokenIntoMultipleLines() throws Exception { + ContinuationHit hit = new ContinuationHit(createMultiBucketProgress()); + assertTrue(hit.getValue().length() > 76); // Ensure we exceed MIME line length limits. + assertFalse(hit.getValue().contains("\n")); + } + + @Test + public void decodingAcceptsUrlSafeTokens() throws Exception { + final ProgressToken token = ContinuationHit.getToken(SINGLE_BUCKET_URL_SAFE_BASE64); + // Roundtrip should yield identical results. + assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, + new ContinuationHit(token).getValue()); + } + + /** + * Legacy Base64 encoder emitted MIME Base64. Ensure we handle tokens from that era. + */ + @Test + public void decodingAcceptsLegacyNonUrlSafeTokens() throws Exception { + final String legacyBase64 = convertedToMimeBase64Chars(SINGLE_BUCKET_URL_SAFE_BASE64); + final ProgressToken legacyToken = ContinuationHit.getToken(legacyBase64); + + assertEquals(SINGLE_BUCKET_URL_SAFE_BASE64, + new ContinuationHit(legacyToken).getValue()); + } + + /** + * Legacy Base64 encoder would happily output line breaks after each MIME line + * boundary. Ensure we handle these gracefully. + */ + @Test + public void decodingAcceptsLegacyMimeLineBrokenTokens() throws Exception { + final String multiBucketLegacyToken = + "AAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAPqNFZ4mrz++gAAAAAAAAAA6jRWeJq8/vsA\r\n" + + "AAAAAAAAAOo0VniavP7/AAAAAAAAAAA="; + final ProgressToken legacyToken = ContinuationHit.getToken(multiBucketLegacyToken); + + assertEquals(MULTI_BUCKET_URL_SAFE_BASE64, + new ContinuationHit(legacyToken).getValue()); + } + + /** + * Returns a ProgressToken whose base 64 representation will be _less_ than 76 bytes (MIME line limit) + */ + private ProgressToken createSingleBucketProgress() { + ProgressToken token = new ProgressToken(16); + // Use explicit bucket set so we can better control the binary representation + // of the buckets, and thus the values written as base 64. + Set<BucketId> buckets = new TreeSet<>(); + // This particular bucket ID will contain +/ chars when output as non-URL safe base 64. + buckets.add(new BucketId(58, 0x123456789abcfeffL)); + VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. + return token; + } + + /** + * Returns a ProgressToken whose base 64 representation will be _more_ than 76 bytes (MIME line limit) + */ + private ProgressToken createMultiBucketProgress() { + ProgressToken token = new ProgressToken(16); + Set<BucketId> buckets = new TreeSet<>(); + buckets.add(new BucketId(58, 0x123456789abcfeffL)); + buckets.add(new BucketId(58, 0x123456789abcfefaL)); + buckets.add(new BucketId(58, 0x123456789abcfefbL)); + VisitorIterator.createFromExplicitBucketSet(buckets, 16, token); // "Prime" the token. + return token; + } + + private String convertedToMimeBase64Chars(String token) { + // Doesn't split on MIME line boundaries, so not fully MIME compliant. + return token.replace('-', '+').replace('_', '/'); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java new file mode 100755 index 00000000000..ea88128aaa9 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DocumentSessionFactory.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.feedapi.DummySessionFactory; +import com.yahoo.feedapi.SendSession; +import com.yahoo.jdisc.Metric; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.HashMap; + +/** + * Used to automatically reply with a GetDocumentReply for every received GetDocumentMessage + */ +public class DocumentSessionFactory extends DummySessionFactory { + + private DocumentType docType; + private Error error; + // Reply instances are shared between the two collections. + List<GetDocumentReply> autoReplies; + Map<DocumentId, GetDocumentReply> autoReplyLookup = new HashMap<>(); + boolean autoReply = true; + boolean nullReply = false; + private int sessionsCreated = 0; + + private class DocumentReplySession extends SendSession { + + ReplyHandler handler; + Error e; + DummySessionFactory owner; + + public DocumentReplySession(ReplyHandler handler, Error e, DummySessionFactory owner) { + this.handler = handler; + this.e = e; + this.owner = owner; + } + + protected Result onSend(Message m, boolean blockIfQueueFull) throws InterruptedException { + if (!(m instanceof GetDocumentMessage)) { + throw new IllegalArgumentException("Expected GetDocumentMessage"); + } + GetDocumentMessage gm = (GetDocumentMessage)m; + owner.messages.add(m); + if (autoReply) { + Document replyDoc; + if (!nullReply) { + replyDoc = new Document(docType, gm.getDocumentId()); + } else { + replyDoc = null; + } + Reply r = new GetDocumentReply(replyDoc); + r.setMessage(m); + r.setContext(m.getContext()); + if (e != null) { + r.addError(e); + } + handler.handleReply(r); + } else if (owner.messages.size() == autoReplies.size()) { + // Pair up all replies with their messages + for (Message msg : owner.messages) { + GetDocumentReply reply = autoReplyLookup.get(((GetDocumentMessage)msg).getDocumentId()); + reply.setMessage(msg); + reply.setContext(msg.getContext()); + if (e != null) { + reply.addError(e); + } + } + // Now send them in the correct order. Instances are shared, so source + // messages and contexts are properly set + for (Reply reply : autoReplies) { + handler.handleReply(reply); + } + } + + return Result.ACCEPTED; + } + + public void close() { + } + } + + public DocumentSessionFactory(DocumentType docType) { + this.docType = docType; + this.error = null; + } + + public DocumentSessionFactory(DocumentType docType, Error error, boolean autoReply, GetDocumentReply... autoReplies) { + this.docType = docType; + this.error = error; + this.autoReplies = Arrays.asList(autoReplies); + for (GetDocumentReply reply : autoReplies) { + this.autoReplyLookup.put(reply.getDocument().getId(), reply); + } + this.autoReply = autoReply; + } + + public boolean isNullReply() { + return nullReply; + } + + public void setNullReply(boolean nullReply) { + this.nullReply = nullReply; + } + + public int getSessionsCreated() { + return sessionsCreated; + } + + public SendSession createSendSession(ReplyHandler r, Metric metric) { + ++sessionsCreated; + return new DocumentReplySession(r, error, this); + } + + @Override + public VisitorSession createVisitorSession(VisitorParameters p) { + return new DummyVisitorSession(p, docType); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java new file mode 100644 index 00000000000..d1c55c0968f --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/DummyVisitorSession.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.documentapi.*; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.Trace; + +import java.util.ArrayList; +import java.util.List; + +/** + * Stub to test visitors. + */ +public class DummyVisitorSession implements VisitorSession { + + final VisitorParameters parameters; + final DocumentType documentType; + final List<Message> autoReplyMessages = new ArrayList<>(); + + DummyVisitorSession(VisitorParameters p, DocumentType documentType) { + parameters = p; + this.documentType = documentType; + p.getLocalDataHandler().setSession(this); + addDefaultReplyMessages(); + } + + @Override + public boolean isDone() { + return true; + } + + @Override + public ProgressToken getProgress() { + return new ProgressToken(12); + } + + @Override + public Trace getTrace() { + return null; + } + + public void addDocumentReply(String docId) { + Document replyDoc = new Document(documentType, docId); + autoReplyMessages.add(new PutDocumentMessage(new DocumentPut(replyDoc))); + } + + public void addRemoveReply(String docId) { + autoReplyMessages.add(new RemoveDocumentMessage(new DocumentId(docId))); + } + + public void addDefaultReplyMessages() { + addDocumentReply("userdoc:foo:1234:bar"); + if (parameters.visitRemoves()) { + addRemoveReply("userdoc:foo:1234:removed"); + } + } + + public void clearAutoReplyMessages() { + autoReplyMessages.clear(); + } + + @Override + public boolean waitUntilDone(long l) throws InterruptedException { + for (Message msg : autoReplyMessages) { + parameters.getLocalDataHandler().onMessage(msg, new AckToken(this)); + } + return true; + } + + @Override + public void ack(AckToken ackToken) { + } + + @Override + public void abort() { + } + + @Override + public VisitorResponse getNext() { + return null; + } + + @Override + public VisitorResponse getNext(int i) throws InterruptedException { + return null; + } + + @Override + public void destroy() { + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java new file mode 100755 index 00000000000..1e39b9766b1 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/GetSearcherTestCase.java @@ -0,0 +1,1090 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.component.chain.Chain; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.jdisc.HeaderFields; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.document.*; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.Raw; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.java7compat.Util; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.prelude.templates.SearchRendererAdaptor; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vespaclient.ClusterList; +import com.yahoo.vespaclient.config.FeederConfig; + +import org.junit.Test; + +import java.io.*; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.zip.GZIPOutputStream; + +import static org.junit.Assert.*; + +public class GetSearcherTestCase { + + private DocumentTypeManager docMan = null; + private DocumentType docType; + private FeederConfig defFeedCfg = new FeederConfig(new FeederConfig.Builder()); + private LoadTypeConfig defLoadTypeCfg = new LoadTypeConfig(new LoadTypeConfig.Builder()); + + @org.junit.Before + public void setUp() { + docMan = new DocumentTypeManager(); + docType = new DocumentType("kittens"); + docType.addHeaderField("name", DataType.STRING); + docType.addField("description", DataType.STRING); + docType.addField("image", DataType.STRING); + docType.addField("fluffiness", DataType.INT); + docType.addField("foo", DataType.RAW); + docMan.registerDocumentType(docType); + } + + @org.junit.After + public void tearDown() { + docMan = null; + docType = null; + } + + private void assertHits(HitGroup hits, String... wantedHits) { + assertEquals(wantedHits.length, hits.size()); + for (int i = 0; i < wantedHits.length; ++i) { + assertTrue(hits.get(i) instanceof DocumentHit); + DocumentHit hit = (DocumentHit)hits.get(i); + assertEquals(wantedHits[i], hit.getDocument().getId().toString()); + } + } + + @Test + public void testGetSingleDocumentQuery() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:1:2")); + System.out.println("HTTP request is " + result.getQuery().getHttpRequest()); + + assertEquals(1, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + } + assertEquals(1, result.hits().size()); + assertHits(result.hits(), "userdoc:kittens:1:2"); + // By default, document hit should not have its hit fields set + DocumentHit hit = (DocumentHit)result.hits().get(0); + assertEquals(0, hit.fieldKeys().size()); + } + + @Test + public void testGetMultipleDocumentsQuery() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Query query = newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + + assertEquals(2, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + } + + { + Message m = factory.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:3:4", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + } + assertEquals(2, result.hits().size()); + assertNull(result.hits().getErrorHit()); + assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4"); + assertEquals(2, query.getHits()); + } + + // Test that you can use both query string and POSTed IDs + @Test + public void testGetMultipleDocumentsQueryAndPOST() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; + MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4"); + Query query = new Query(request.toRequest()); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + + assertEquals(5, factory.messages.size()); + assertEquals(5, result.hits().size()); + assertNull(result.hits().getErrorHit()); + assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", + "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); + } + + @Test + public void testGetMultipleDocumentsQueryAndGZippedPOST() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + String data = "userdoc:kittens:5:6\nuserdoc:kittens:7:8\nuserdoc:kittens:9:10"; + + // Create with automatic GZIP encoding + MockHttpRequest request = new MockHttpRequest(data.getBytes("utf-8"), "/get/?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4", true); + Query query = new Query(request.toRequest()); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + + assertEquals(5, factory.messages.size()); + assertEquals(5, result.hits().size()); + assertNull(result.hits().getErrorHit()); + assertHits(result.hits(), "userdoc:kittens:1:2", "userdoc:kittens:3:4", + "userdoc:kittens:5:6", "userdoc:kittens:7:8", "userdoc:kittens:9:10"); + } + + /* Test that a query without any ids is passed through to the next chain */ + @Test + public void testQueryPassThrough() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + HitGroup hits = new HitGroup("mock"); + hits.add(new Hit("blernsball")); + Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern")); + + assertEquals(0, factory.messages.size()); + assertEquals(1, result.hits().size()); + assertNotNull(result.hits().get("blernsball")); + } + + /* Test that a query will contain both document hits and hits from a searcher + * further down the chain, iff the searcher returns a DocumentHit. + */ + @Test + public void testQueryPassThroughAndGet() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); + doc1.setFieldValue("name", new StringFieldValue("megacat")); + doc1.setFieldValue("description", new StringFieldValue("supercat")); + doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1) + }; + + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + DocumentHit backendHit = new DocumentHit(new Document(docType, new DocumentId("userdoc:kittens:5678:bar")), 5); + Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(backendHit)); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?query=flarn&id=userdoc:kittens:1234:foo")); + + assertEquals(1, factory.messages.size()); + assertEquals(2, result.hits().size()); + assertNotNull(result.hits().get("userdoc:kittens:5678:bar")); + assertNotNull(result.hits().get("userdoc:kittens:1234:foo")); + } + + @Test + public void testQueryPassThroughAndGetUnknownBackendHit() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + HitGroup hits = new HitGroup("mock"); + hits.add(new Hit("blernsball")); + Chain<Searcher> searchChain = new Chain<>(searcher, new MockBackend(hits)); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?flarn=blern&id=userdoc:kittens:9:aaa")); + + assertEquals(0, factory.messages.size()); + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + + "A backend searcher to com.yahoo.storage.searcher.GetSearcher returned a " + + "hit that was not an instance of com.yahoo.storage.searcher.DocumentHit. " + + "Only DocumentHit instances are supported in the backend hit result set " + + "when doing queries that contain document identifier sets recognised by the " + + "Get Searcher.\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testConfig() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("route66").retryenabled(false)), defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); + + assertEquals(1, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("doc:batman:dahnahnahnah", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(Route.parse("route66"), gdm.getRoute()); + assertFalse(gdm.getRetryEnabled()); + assertEquals(458000, gdm.getTimeRemaining()); + } + } + + @Test + public void testConfigChanges() throws Exception { + String config = "raw:timeout 458\nroute \"riksveg18\"\nretryenabled true"; + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)), + defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=doc:batman:dahnahnahnah")); + + assertEquals(1, factory.messages.size()); + assertEquals(1, factory.getSessionsCreated()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("doc:batman:dahnahnahnah", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(Route.parse("riksveg18"), gdm.getRoute()); + assertTrue(gdm.getRetryEnabled()); + assertEquals(458000, gdm.getTimeRemaining()); + } + + factory.messages.clear(); + + FeederConfig newConfig = new FeederConfig(new FeederConfig.Builder() + .timeout(123) + .route("e6") + .retryenabled(false) + ); + searcher.getMessagePropertyProcessor().configure(newConfig, defLoadTypeCfg); + + new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=doc:spiderman:does_whatever_a_spider_can")); + + // riksveg18 is created again, and e6 is created as well. + assertEquals(3, factory.getSessionsCreated()); + + assertEquals(1, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("doc:spiderman:does_whatever_a_spider_can", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(Route.parse("e6"), gdm.getRoute()); + assertFalse(gdm.getRetryEnabled()); + assertEquals(123000, gdm.getTimeRemaining()); + } + } + + @Test + public void testQueryOverridesDefaults() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=458")); + + assertEquals(2, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); + assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); + assertEquals(458000, gdm.getTimeRemaining()); + } + + { + Message m = factory.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:3:4", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); + assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); + assertEquals(458000, gdm.getTimeRemaining()); + } + } + + @Test + public void testQueryOverridesConfig() throws Exception { + String config = "raw:timeout 458\nroute \"route66\""; + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&priority=LOW_2&route=highwaytohell&timeout=123")); + + assertEquals(2, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); + assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); + assertEquals(123000, gdm.getTimeRemaining()); + } + + { + Message m = factory.messages.get(1); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:3:4", d.toString()); + assertEquals("[all]", gdm.getFieldSet()); + assertEquals(DocumentProtocol.Priority.LOW_2, gdm.getPriority()); + assertEquals(Route.parse("highwaytohell"), gdm.getRoute()); + assertEquals(123000, gdm.getTimeRemaining()); + } + } + + @Test + public void testBadPriorityValue() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1:2&priority=onkel_jubalon")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: No enum const" + + (Util.isJava7Compatible() ? "ant " : " class ") + + "com.yahoo.documentapi.messagebus.protocol.DocumentProtocol" + + (Util.isJava7Compatible() ? "." : "$") + + "Priority.onkel_jubalon\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testMultiIdBadArrayIndex() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + { + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[1]=userdoc:kittens:1:2")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: query contains document ID " + + "array that is not zero-based and/or linearly increasing\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + { + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[2]=userdoc:kittens:2:3")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: query contains document ID " + + "array that is not zero-based and/or linearly increasing\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + { + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1:2&id[1]=userdoc:kittens:2:3")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: query contains document ID " + + "array that is not zero-based and/or linearly increasing\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + { + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0=userdoc:kittens:1:2")); + + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: Malformed document ID array parameter\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + } + + @Test + public void testLegacyHeadersOnly() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1:2&headersonly=true")); + + assertEquals(1, factory.messages.size()); + { + Message m = factory.messages.get(0); + assertEquals(DocumentProtocol.MESSAGE_GETDOCUMENT, m.getType()); + GetDocumentMessage gdm = (GetDocumentMessage)m; + DocumentId d = gdm.getDocumentId(); + assertEquals("userdoc:kittens:1:2", d.toString()); + assertEquals("[header]", gdm.getFieldSet()); + } + assertEquals(1, result.hits().size()); + assertHits(result.hits(), "userdoc:kittens:1:2"); + } + + @Test + public void testFieldSet() throws Exception { + } + + @Test + public void testConsistentResultOrdering() throws Exception { + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:1:2"))), + new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:7:8"))), + new GetDocumentReply(new Document(docType, new DocumentId("userdoc:kittens:555:123"))) + }; + + // Use a predefined reply list to ensure messages are answered out of order + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:555:123&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:7:8")); + + assertEquals(3, factory.messages.size()); + assertEquals(3, result.hits().size()); + // Hits must be in the same order as their document IDs in the query + assertHits(result.hits(), "userdoc:kittens:555:123", "userdoc:kittens:1:2", "userdoc:kittens:7:8"); + + assertEquals(0, ((DocumentHit)result.hits().get(0)).getIndex()); + assertEquals(1, ((DocumentHit)result.hits().get(1)).getIndex()); + assertEquals(2, ((DocumentHit)result.hits().get(2)).getIndex()); + } + + @Test + public void testResultWithSingleError() throws Exception { + com.yahoo.messagebus.Error err = new com.yahoo.messagebus.Error(32, "Alas, it went poorly"); + DocumentSessionFactory factory = new DocumentSessionFactory(docType, err, true); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4")); + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"messagebus\" code=\"32\" message=\"Alas, it went poorly\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testResultWithMultipleErrors() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); + Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:99:111")); + GetDocumentReply errorReply1 = new GetDocumentReply(doc1); + errorReply1.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); + GetDocumentReply errorReply2 = new GetDocumentReply(doc2); + errorReply2.addError(new com.yahoo.messagebus.Error(456, "userdoc:kittens:99:111 shredded the curtains.")); + GetDocumentReply[] replies = new GetDocumentReply[] { + errorReply1, + errorReply2 + }; + + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:99:111")); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + + "<error type=\"messagebus\" code=\"456\" message=\"userdoc:kittens:99:111 shredded the curtains.\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testResultWithNullDocument() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); + factory.setNullReply(true); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:55:bad_document_id")); + // Document not found does not produce any hit at all, error or otherwise + assertNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "</result>\n", result); + } + + @Test + public void testDefaultDocumentHitRendering() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); + doc1.setFieldValue("name", new StringFieldValue("mittens")); + doc1.setFieldValue("description", new StringFieldValue("it's a cat")); + doc1.setFieldValue("fluffiness", new IntegerFieldValue(8)); + Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); + doc2.setFieldValue("name", new StringFieldValue("garfield")); + doc2.setFieldValue("description", + new StringFieldValue("preliminary research indicates <em>hatred</em> of mondays. caution advised")); + doc2.setFieldValue("fluffiness", new IntegerFieldValue(2)); + Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); + GetDocumentReply errorReply = new GetDocumentReply(doc3); + errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 had fleas.")); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + new GetDocumentReply(doc2), + errorReply + }; + + // Use a predefined reply list to ensure messages are answered out of order + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result xmlResult = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 had fleas.\"/>\n" + + "</errors>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + + " <name>garfield</name>\n" + + " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + + " <fluffiness>2</fluffiness>\n" + + "</document>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + + " <name>mittens</name>\n" + + " <description>it's a cat</description>\n" + + " <fluffiness>8</fluffiness>\n" + + "</document>\n" + + "</result>\n", xmlResult); + } + + @Test + public void testDocumentFieldNoContentType() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + doc1.setFieldValue("name", "derrick"); + doc1.setFieldValue("description", "kommisar katze"); + doc1.setFieldValue("fluffiness", 0); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:5:1&field=description")); + + assertNull(result.hits().getErrorHit()); + assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); + assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>kommisar katze</result>\n", result); + } + + @Test + public void testDocumentFieldEscapeXML() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + doc1.setFieldValue("name", "asfd"); + doc1.setFieldValue("description", "<script type=\"evil/madness\">horror & screams</script>"); + doc1.setFieldValue("fluffiness", 0); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:5:1&field=description")); + + assertNull(result.hits().getErrorHit()); + assertEquals("text/xml", result.getTemplating().getTemplates().getMimeType()); + assertEquals("UTF-8", result.getTemplating().getTemplates().getEncoding()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result><script type=\"evil/madness\">horror & screams</script></result>\n", result); + } + + @Test + public void testDocumentFieldRawContent() throws Exception { + byte[] contentBytes = new byte[] { 0, -128, 127 }; + + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); + doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1) + }; + + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:123:456&field=foo")); + + assertNull(result.hits().getErrorHit()); + assertEquals("application/octet-stream", result.getTemplating().getTemplates().getMimeType()); + + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + SearchRendererAdaptor.callRender(stream, result); + stream.flush(); + + byte[] resultBytes = stream.toByteArray(); + assertEquals(contentBytes.length, resultBytes.length); + for (int i = 0; i < resultBytes.length; ++i) { + assertEquals(contentBytes[i], resultBytes[i]); + } + } + + @Test + public void testDocumentFieldRawWithContentOverride() throws Exception { + byte[] contentBytes = new byte[] { 0, -128, 127 }; + + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:123:456")); + doc1.setFieldValue("foo", new Raw(ByteBuffer.wrap(contentBytes))); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1) + }; + + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:123:456&field=foo&contenttype=text/fancy")); + + assertNull(result.hits().getErrorHit()); + assertEquals("text/fancy", result.getTemplating().getTemplates().getMimeType()); + + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + SearchRendererAdaptor.callRender(stream, result); + stream.flush(); + + byte[] resultBytes = stream.toByteArray(); + assertEquals(contentBytes.length, resultBytes.length); + for (int i = 0; i < resultBytes.length; ++i) { + assertEquals(contentBytes[i], resultBytes[i]); + } + } + + @Test + public void testDocumentFieldWithMultipleIDs() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:1:2&id[1]=userdoc:kittens:3:4&field=name")); + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"3\" message=\"Illegal query: " + + "java.lang.IllegalArgumentException: Field only valid for single document id query\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testDocumentFieldNotSet() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + doc1.setFieldValue("name", "asdf"); + doc1.setFieldValue("description", "fdsafsdf"); + doc1.setFieldValue("fluffiness", 10); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:5:1&field=image")); + + assertNotNull(result.hits().getErrorHit()); + assertEquals(1, result.hits().size()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + + "Field 'image' found in document type, but had no content in userdoc:kittens:5:1\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + + @Test + public void testDocumentFieldWithDocumentNotFound() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, true); + factory.setNullReply(true); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1:2&field=name")); + assertNotNull(result.hits().getErrorHit()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"16\" message=\"Resource not found.: " + + "Document not found, could not return field 'name'\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testDocumentFieldNotReachableWithHeadersOnly() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + doc1.setFieldValue("name", "asdf"); + // don't set body fields + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:5:1&field=description&headersonly=true")); + + assertNotNull(result.hits().getErrorHit()); + assertEquals(1, result.hits().size()); + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"4\" message=\"Invalid query parameter: " + + "Field 'description' is located in document body, but headersonly " + + "prevents it from being retrieved in userdoc:kittens:5:1\"/>\n" + + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testVespaXMLTemplate() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:3:4")); + doc1.setFieldValue("name", "mittens"); + doc1.setFieldValue("description", "it's a cat"); + doc1.setFieldValue("fluffiness", 8); + Document doc2 = new Document(docType, new DocumentId("userdoc:kittens:1:2")); + doc2.setFieldValue("name", "garfield"); + doc2.setFieldValue("description", "preliminary research indicates <em>hatred</em> of mondays. caution advised"); + doc2.setFieldValue("fluffiness", 2); + Document doc3 = new Document(docType, new DocumentId("userdoc:kittens:77:88")); + GetDocumentReply errorReply = new GetDocumentReply(doc3); + errorReply.addError(new com.yahoo.messagebus.Error(123, "userdoc:kittens:77:88 lost in a <ni!>\"shrubbery\"</ni!>")); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1), + new GetDocumentReply(doc2), + errorReply + }; + + // Use a predefined reply list to ensure messages are answered out of order + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id[0]=userdoc:kittens:77:88&id[1]=userdoc:kittens:1:2&id[2]=userdoc:kittens:3:4")); // TODO! + + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"messagebus\" code=\"123\" message=\"userdoc:kittens:77:88 lost in a <ni!>"shrubbery"</ni!>\"/>\n"+ + "</errors>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:1:2\">\n" + + " <name>garfield</name>\n" + + " <description>preliminary research indicates <em>hatred</em> of mondays. caution advised</description>\n" + + " <fluffiness>2</fluffiness>\n" + + "</document>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:kittens:3:4\">\n" + + " <name>mittens</name>\n" + + " <description>it's a cat</description>\n" + + " <fluffiness>8</fluffiness>\n" + + "</document>\n" + + "</result>\n", result); + } + + @Test + public void testDocumentHitWithPopulatedHitFields() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:1234:foo")); + doc1.setFieldValue("name", new StringFieldValue("megacat")); + doc1.setFieldValue("description", new StringFieldValue("supercat")); + doc1.setFieldValue("fluffiness", new IntegerFieldValue(10000)); + GetDocumentReply[] replies = new GetDocumentReply[] { + new GetDocumentReply(doc1) + }; + + // Use a predefined reply list to ensure messages are answered out of order + Chain<Searcher> searchChain = createSearcherChain(replies); + + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("?id=userdoc:kittens:1234:foo&populatehitfields=true")); + assertEquals(1, result.hits().size()); + assertHits(result.hits(), "userdoc:kittens:1234:foo"); + + DocumentHit hit = (DocumentHit)result.hits().get(0); + Iterator<Map.Entry<String, Object>> iter = hit.fieldIterator(); + Set<String> fieldSet = new TreeSet<>(); + while (iter.hasNext()) { + Map.Entry<String, Object> kv = iter.next(); + StringBuilder field = new StringBuilder(); + field.append(kv.getKey()).append(" -> ").append(kv.getValue()); + fieldSet.add(field.toString()); + } + StringBuilder fields = new StringBuilder(); + for (String s : fieldSet) { + fields.append(s).append("\n"); + } + assertEquals( + "description -> supercat\n" + + "documentid -> userdoc:kittens:1234:foo\n" + + "fluffiness -> 10000\n" + + "name -> megacat\n", + fields.toString()); + } + + @Test + public void deserializationExceptionsAreHandledGracefully() throws Exception { + Document doc1 = new Document(docType, new DocumentId("userdoc:kittens:5:1")); + GetDocumentReply[] replies = new GetDocumentReply[] { + new MockFailingGetDocumentReply(doc1), + }; + Chain<Searcher> searchChain = createSearcherChain(replies); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("?id=userdoc:kittens:5:1")); + assertRendered("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<errors>\n" + + "<error type=\"searcher\" code=\"18\" message=\"Internal server error.: " + + "Got exception of type java.lang.RuntimeException during document " + + "deserialization: epic dragon attack\"/>\n"+ + "</errors>\n" + + "</result>\n", result); + } + + @Test + public void testJsonRendererSetting() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); // Needs auto-reply + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + Chain<Searcher> searchChain = new Chain<>(searcher); + + Query query = newQuery("?id=userdoc:kittens:1:2&format=json"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + assertFalse(result.getTemplating().getTemplates() instanceof DocumentXMLTemplate); + } + + private Query newQuery(String queryString) { + return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + } + + private Chain<Searcher> createSearcherChain(GetDocumentReply[] replies) throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType, null, false, replies); + GetSearcher searcher = new GetSearcher(new FeedContext( + new MessagePropertyProcessor(defFeedCfg, defLoadTypeCfg), + factory, docMan, new ClusterList(), new NullFeedMetric())); + return new Chain<>(searcher); + } + + private static class MockFailingGetDocumentReply extends GetDocumentReply { + private int countdown = 2; + + private MockFailingGetDocumentReply(Document doc) { + super(doc); + } + + @Override + public Document getDocument() { + // Reason for countdown is that the test DocumentSessionFactory calls + // getDocument once internally before code can ever reach handleReply. + if (--countdown == 0) { + throw new RuntimeException("epic dragon attack"); + } + return super.getDocument(); + } + } + + private static class MockBackend extends Searcher { + private Hit hitToReturn; + + public MockBackend(Hit hitToReturn) { + this.hitToReturn = hitToReturn; + } + + public @Override Result search(Query query, Execution execution) { + Result result = new Result(query); + result.hits().add(hitToReturn); + return result; + } + } + + private class MockHttpRequest { + + private final String req; + private byte[] data; + private boolean gzip = false; + + MockHttpRequest(byte[] data, String req) { + this.req = req; + this.data = data; + } + + MockHttpRequest(byte[] data, String req, boolean gzip) { + this.data = data; + this.req = req; + this.gzip = gzip; + } + + public InputStream getData() { + if (gzip) { + try { + ByteArrayOutputStream rawOut = new ByteArrayOutputStream(); + GZIPOutputStream compressed = new GZIPOutputStream(rawOut); + compressed.write(data, 0, data.length); + compressed.finish(); + compressed.flush(); + rawOut.flush(); + return new ByteArrayInputStream(rawOut.toByteArray()); + } catch (Exception e) { + return null; + } + } + return new ByteArrayInputStream(data); + } + + public void addHeaders(HeaderFields headers) { + headers.add("Content-Type", "text/plain;encoding=UTF-8"); + if (gzip) + headers.add("Content-Encoding", "gzip"); + } + + public com.yahoo.container.jdisc.HttpRequest toRequest() { + com.yahoo.container.jdisc.HttpRequest request = com.yahoo.container.jdisc.HttpRequest.createTestRequest(req, com.yahoo.jdisc.http.HttpRequest.Method.GET, getData()); + addHeaders(request.getJDiscRequest().headers()); + return request; + } + + } + + public static void assertRendered(String expected,Result result) throws Exception { + assertRendered(expected,result,true); + } + + public static void assertRendered(String expected,Result result,boolean checkFullEquality) throws Exception { + if (checkFullEquality) + assertEquals(expected, ResultRenderingUtil.getRendered(result)); + else + assertTrue(ResultRenderingUtil.getRendered(result).startsWith(expected)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java new file mode 100644 index 00000000000..21d39089d2a --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/ResultRenderingUtil.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.prelude.templates.SearchRendererAdaptor; +import com.yahoo.processing.rendering.Renderer; +import com.yahoo.search.Result; +import com.yahoo.search.rendering.DefaultRenderer; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; + +public class ResultRenderingUtil { + + public static String getRendered(Result result) throws Exception { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + Charset cs = Charset.forName("utf-8"); + CharsetDecoder decoder = cs.newDecoder(); + SearchRendererAdaptor.callRender(stream, result); + stream.flush(); + return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString(); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java new file mode 100644 index 00000000000..820f7f56e2f --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java @@ -0,0 +1,248 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.storage.searcher; + +import com.yahoo.component.chain.Chain; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.protect.Error; +import com.yahoo.documentapi.VisitorSession; +import com.yahoo.feedhandler.NullFeedMetric; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.documentapi.VisitorParameters; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.feedapi.FeedContext; +import com.yahoo.feedapi.MessagePropertyProcessor; +import com.yahoo.messagebus.StaticThrottlePolicy; +import com.yahoo.prelude.templates.DefaultTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vdslib.VisitorOrdering; +import com.yahoo.vespaclient.ClusterList; +import com.yahoo.vespaclient.config.FeederConfig; + +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class VisitorSearcherTestCase { + + private DocumentTypeManager docMan = null; + private DocumentType docType; + DocumentSessionFactory factory; + + @org.junit.Before + public void setUp() { + docMan = new DocumentTypeManager(); + docType = new DocumentType("kittens"); + docType.addHeaderField("name", DataType.STRING); + docType.addField("description", DataType.STRING); + docType.addField("image", DataType.RAW); + docType.addField("fluffiness", DataType.INT); + docType.addField("foo", DataType.RAW); + docMan.registerDocumentType(docType); + factory = new DocumentSessionFactory(docType); + } + + public VisitSearcher create() throws Exception { + ClusterListConfig.Storage.Builder storageCluster = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foobar").name("foobar"); + ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(storageCluster)); + ClusterList clusterList = new ClusterList(); + clusterList.configure(clusterListCfg); + return new VisitSearcher(new FeedContext( + new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)), + new LoadTypeConfig(new LoadTypeConfig.Builder())), + factory, docMan, clusterList, new NullFeedMetric())); + } + + @Test + public void testQueryParameters() throws Exception { + VisitSearcher searcher = create(); + VisitorParameters params = searcher.getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234&visit.cluster=foobar" + + "&visit.dataHandler=othercluster&visit.fieldSet=[header]&visit.fromTimestamp=112&visit.toTimestamp=224" + + "&visit.maxBucketsPerVisitor=2&visit.maxPendingMessagesPerVisitor=7&visit.maxPendingVisitors=14" + + "&visit.ordering=ASCENDING&priority=NORMAL_1&tracelevel=7&visit.visitInconsistentBuckets&visit.visitRemoves"), null); + + assertEquals("id.user=1234", params.getDocumentSelection()); + assertEquals(7, params.getMaxPending()); + assertEquals(2, params.getMaxBucketsPerVisitor()); + assertEquals(14, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); + assertEquals("[Storage:cluster=foobar;clusterconfigid=storage/cluster.foobar]", params.getRoute().toString()); + assertEquals("othercluster", params.getRemoteDataHandler()); + assertEquals("[header]", params.fieldSet()); + assertEquals(112, params.getFromTimestamp()); + assertEquals(224, params.getToTimestamp()); + assertEquals(VisitorOrdering.ASCENDING, params.getVisitorOrdering()); + assertEquals(DocumentProtocol.Priority.NORMAL_1, params.getPriority()); + assertEquals(7, params.getTraceLevel()); + assertEquals(true, params.visitInconsistentBuckets()); + assertEquals(true, params.visitRemoves()); + } + + @Test + public void timestampQueryParametersAreParsedAsLongs() throws Exception { + VisitorParameters params = create().getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234&" + + "visit.fromTimestamp=1419021596000000&" + + "visit.toTimestamp=1419021597000000"), null); + assertEquals(1419021596000000L, params.getFromTimestamp()); + assertEquals(1419021597000000L, params.getToTimestamp()); + } + + @Test + public void testQueryParametersDefaults() throws Exception { + VisitSearcher searcher = create(); + VisitorParameters params = searcher.getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234&hits=100"), null); + + assertEquals("id.user=1234", params.getDocumentSelection()); + assertEquals(1, params.getMaxBucketsPerVisitor()); + assertEquals(1, ((StaticThrottlePolicy)params.getThrottlePolicy()).getMaxPendingCount()); + assertEquals(1, params.getMaxFirstPassHits()); + assertEquals(1, params.getMaxTotalHits()); + assertEquals(32, params.getMaxPending()); + assertEquals(false, params.visitInconsistentBuckets()); + } + + @Test + public void testWrongCluster() throws Exception { + VisitSearcher searcher = create(); + + try { + searcher.getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234&visit.cluster=unknown"), null); + + assertTrue(false); + } catch (Exception e) { + // e.printStackTrace(); + } + } + + + @Test(expected = IllegalArgumentException.class) + public void testNoClusterParamWhenSeveralClusters() throws Exception { + DocumentSessionFactory factory = new DocumentSessionFactory(docType); + ClusterListConfig.Storage.Builder storageCluster1 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foo").name("foo"); + ClusterListConfig.Storage.Builder storageCluster2 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.bar").name("bar"); + ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(Arrays.asList(storageCluster1, storageCluster2))); + ClusterList clusterList = new ClusterList(); + clusterList.configure(clusterListCfg); + VisitSearcher searcher = new VisitSearcher(new FeedContext( + new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(100).route("whatever").retryenabled(true)), + new LoadTypeConfig(new LoadTypeConfig.Builder())), + factory, docMan, clusterList, new NullFeedMetric())); + + searcher.getVisitorParameters( + newQuery("visit?visit.selection=id.user=1234"), null); + } + + @Test + public void testSimple() throws Exception { + Chain<Searcher> searchChain = new Chain<>(create()); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(newQuery("visit?visit.selection=id.user=1234&hits=100")); + assertEquals(1, result.hits().size()); + assertRendered( + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + + "</result>\n", result); + } + + private Result invokeVisitRemovesSearchChain() throws Exception { + Chain<Searcher> searchChain = new Chain<>(create()); + return new Execution(searchChain, Execution.Context.createContextStub()).search( + newQuery("visit?visit.selection=id.user=1234&hits=100&visit.visitRemoves=true")); + } + + @Test + public void visitRemovesIncludesRemoveEntriesInResultXml() throws Exception { + Result result = invokeVisitRemovesSearchChain(); + assertEquals(2, result.hits().size()); + assertRendered( + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<document documenttype=\"kittens\" documentid=\"userdoc:foo:1234:bar\"/>\n" + + "<remove documentid=\"userdoc:foo:1234:removed\"/>\n" + + "</result>\n", result); + } + + @Test + public void removedDocumentIdsAreXmlEscaped() throws Exception { + factory = mock(DocumentSessionFactory.class); + when(factory.createVisitorSession(any(VisitorParameters.class))).thenAnswer((p) -> { + VisitorParameters params = (VisitorParameters)p.getArguments()[0]; + DummyVisitorSession session = new DummyVisitorSession(params, docType); + session.clearAutoReplyMessages(); + session.addRemoveReply("userdoc:foo:1234:<rem\"o\"ved&stuff>"); + return session; + }); + Result result = invokeVisitRemovesSearchChain(); + assertEquals(1, result.hits().size()); + assertRendered( + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + + "<result>\n" + + "<remove documentid=\"userdoc:foo:1234:<rem"o"ved&stuff>\"/>\n" + + "</result>\n", result); + } + + private Result invokeSearcherWithUserQuery() throws Exception { + Chain<Searcher> searchChain = new Chain<>(create()); + return new Execution(searchChain, Execution.Context.createContextStub()) + .search(new Query("visit?visit.selection=id.user=1234&hits=100")); + } + + @Test + public void waitUntilDoneFailureReturnsTimeoutErrorHit() throws Exception { + VisitorSession session = mock(VisitorSession.class); + when(session.waitUntilDone(anyLong())).thenReturn(false); + factory = mock(DocumentSessionFactory.class); + when(factory.createVisitorSession(any(VisitorParameters.class))).thenReturn(session); + + Result result = invokeSearcherWithUserQuery(); + assertNotNull(result.hits().getErrorHit()); + assertEquals(Error.TIMEOUT.code, result.hits().getErrorHit().errors().iterator().next().getCode()); + } + + @Test + public void testRendererWiring() throws Exception { + Chain<Searcher> searchChain = new Chain<>(create()); + { + Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=json"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + assertEquals(DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); + } + { + Query query = newQuery("visit?visit.selection=id.user=1234&hits=100&format=JsonRenderer"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + assertEquals(DefaultTemplateSet.class, result.getTemplating().getTemplates().getClass()); + } + { + Query query = newQuery("visit?visit.selection=id.user=1234&hits=100"); + Result result = new Execution(searchChain, Execution.Context.createContextStub()).search(query); + assertEquals(DocumentXMLTemplate.class, result.getTemplating().getTemplates().getClass()); + } + } + + public static void assertRendered(String expected, Result result) throws Exception { + assertEquals(expected, ResultRenderingUtil.getRendered(result)); + } + + private Query newQuery(String queryString) { + return new Query(HttpRequest.createTestRequest(queryString, com.yahoo.jdisc.http.HttpRequest.Method.GET)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java new file mode 100644 index 00000000000..d19560ee553 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/DummyMetric.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.jdisc.Metric; + +import java.util.Map; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.20 + */ +class DummyMetric implements Metric { + + @Override + public void set(String key, Number val, Context ctx) { + } + + @Override + public void add(String key, Number val, Context ctx) { + } + + @Override + public Context createContext(Map<String, ?> properties) { + return DummyContext.INSTANCE; + } + + private static class DummyContext implements Context { + private static final DummyContext INSTANCE = new DummyContext(); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java new file mode 100644 index 00000000000..2cabed13a7b --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/FeedHandlerCompressionTest.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPOutputStream; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FeedHandlerCompressionTest { + + public static byte[] compress(final String dataToBrCompressed) throws IOException { + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(dataToBrCompressed.length()); + final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream); + gzipOutputStream.write(dataToBrCompressed.getBytes()); + gzipOutputStream.close(); + byte[] compressedBytes = byteArrayOutputStream.toByteArray(); + byteArrayOutputStream.close(); + return compressedBytes; + } + + @Test + public void testUnzipStreamIfNeeded() throws Exception { + final String testData = "foo bar"; + InputStream inputStream = new ByteArrayInputStream(compress(testData)); + HttpRequest httpRequest = mock(HttpRequest.class); + when(httpRequest.getHeader("content-encoding")).thenReturn("gzip"); + InputStream decompressedStream = FeedHandler.unzipStreamIfNeeded(inputStream, httpRequest); + final StringBuilder processedInput = new StringBuilder(); + while (true) { + int readValue = decompressedStream.read(); + if (readValue < 0) { + break; + } + processedInput.append((char)readValue); + } + assertThat(processedInput.toString(), is(testData)); + } + + /** + * Test by setting encoding, but not compressing data. + * @throws Exception + */ + @Test + public void testUnzipFails() throws Exception { + final String testData = "foo bar"; + InputStream inputStream = new ByteArrayInputStream(testData.getBytes()); + HttpRequest httpRequest = mock(HttpRequest.class); + when(httpRequest.getHeader("Content-Encoding")).thenReturn("gzip"); + InputStream decompressedStream = FeedHandler.unzipStreamIfNeeded(inputStream, httpRequest); + final StringBuilder processedInput = new StringBuilder(); + while (true) { + int readValue = decompressedStream.read(); + if (readValue < 0) { + break; + } + processedInput.append((char)readValue); + } + assertThat(processedInput.toString(), is(testData)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java new file mode 100644 index 00000000000..ae96a25fad3 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MetaStream.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.text.Utf8; + +import java.io.ByteArrayInputStream; + +/** + * Stream with extra data outside the actual input stream. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public final class MetaStream extends ByteArrayInputStream { + + private byte[] operations; + int i; + + public MetaStream(byte[] buf) { + super(createPayload(buf)); + this.operations = buf; + i = 0; + } + + private static final byte[] createPayload(byte[] buf) { + StringBuilder bu = new StringBuilder(); + for (byte b : buf) { + bu.append("id:banana:banana::doc1 0\n"); + } + return Utf8.toBytes(bu.toString()); + } + + public byte getNextOperation() { + if (i >= operations.length) { + return 0; + } + return operations[i++]; + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java new file mode 100644 index 00000000000..1208ce334d9 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockNetwork.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import java.util.List; + +import com.yahoo.jrt.slobrok.api.IMirror; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.network.Network; +import com.yahoo.messagebus.network.NetworkOwner; +import com.yahoo.messagebus.routing.RoutingNode; + +/** + * NOP-network. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +class MockNetwork implements Network { + + @Override + public boolean waitUntilReady(double seconds) { + return true; + } + + @Override + public void attach(NetworkOwner owner) { + } + + @Override + public void registerSession(String session) { + } + + @Override + public void unregisterSession(String session) { + + } + + @Override + public boolean allocServiceAddress(RoutingNode recipient) { + return true; + } + + @Override + public void freeServiceAddress(RoutingNode recipient) { + + } + + @Override + public void send(Message msg, List<RoutingNode> recipients) { + } + + @Override + public void sync() { + } + + @Override + public void shutdown() { + } + + @Override + public String getConnectionSpec() { + return null; + } + + @Override + public IMirror getMirror() { + return null; + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java new file mode 100644 index 00000000000..298c925d032 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/MockReply.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.messagebus.Reply; +import com.yahoo.text.Utf8String; + +/** + * Minimal reply simulator. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +class MockReply extends Reply { + + Object context; + + public MockReply(Object context) { + this.context = context; + } + + @Override + public Utf8String getProtocol() { + return null; + } + + @Override + public int getType() { + return 0; + } + + @Override + public Object getContext() { + return context; + } + +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java new file mode 100644 index 00000000000..2e88c440b12 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ErrorsInResultTestCase.java @@ -0,0 +1,236 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.text.Utf8String; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespaxmlparser.MockFeedReaderFactory; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; + +/** + * Check FeedHandler APIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class V2ErrorsInResultTestCase { + + LessConfiguredHandler handler; + ExecutorService workers; + + @Before + public void setUp() throws Exception { + workers = Executors.newCachedThreadPool(); + handler = new LessConfiguredHandler(workers); + } + + @After + public void tearDown() throws Exception { + handler.destroy(); + workers.shutdown(); + } + + private static class LessConfiguredHandler extends FeedHandler { + + public LessConfiguredHandler(Executor executor) throws Exception { + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + } + + + @Override + protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream, + BlockingQueue<OperationStatus> operations, String clientId, + boolean sessionIdWasGeneratedJustNow, int protocolVersion) + throws Exception { + return new LessConfiguredFeeder(requestInputStream, operations, + popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, + sourceSessionParams(request), null, this, this.feedReplyHandler, ""); + } + + @Override + protected DocumentTypeManager createDocumentManager( + DocumentmanagerConfig documentManagerConfig) { + return null; + } + } + + private static class MockSharedSession extends SharedSourceSession { + int count; + + public MockSharedSession(SourceSessionParams params) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + count = 0; + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + Result r; + ReplyHandler handler = msg.popHandler(); + + switch (count++) { + case 0: + r = new Result(ErrorCode.FATAL_ERROR, + "boom"); + break; + case 1: + r = new Result(ErrorCode.TRANSIENT_ERROR, + "transient boom"); + break; + case 2: + final FailedReply reply = new FailedReply(msg.getContext()); + reply.addError(new Error( + ErrorCode.FATAL_ERROR, + "bad mojo, dude")); + handler.handleReply(reply); + r = Result.ACCEPTED; + break; + default: + handler.handleReply(new MockReply(msg.getContext())); + r = Result.ACCEPTED; + } + return r; + } + + } + + private static class FailedReply extends Reply { + Object context; + + public FailedReply(Object context) { + this.context = context; + } + + @Override + public Utf8String getProtocol() { + return null; + } + + @Override + public int getType() { + return 0; + } + + @Override + public Object getContext() { + return context; + } + } + + private static class LessConfiguredFeeder extends Feeder { + + public LessConfiguredFeeder(InputStream stream, + BlockingQueue<OperationStatus> operations, + ClientState storedState, FeederSettings settings, + String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, + SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, + String localHostname) throws Exception { + super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, + sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); + } + + protected ReferencedResource<SharedSourceSession> retainSession( + SourceSessionParams sessionParams, SessionCache sessionCache) { + final SharedSourceSession session = new MockSharedSession(sessionParams); + return new ReferencedResource<>(session, References.fromResource(session)); + } + } + + @Test + public final void test() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 ERROR boom \n", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR transient{20}boom \n", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 ERROR bad{20}mojo,{20}dude \n", + Utf8.toString(out.toByteArray())); + } + + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java new file mode 100644 index 00000000000..74ed844d69f --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2ExternalFeedTestCase.java @@ -0,0 +1,535 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.Result; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.network.Network; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.http.client.config.FeedParams.DataFormat; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespaxmlparser.MockFeedReaderFactory; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * Check FeedHandler APIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class V2ExternalFeedTestCase { + + LessConfiguredHandler handler; + ExecutorService workers; + Level logLevel; + Logger logger; + boolean initUseParentHandlers; + LogBuffer logChecker; + + @Before + public void setUp() throws Exception { + workers = Executors.newCachedThreadPool(); + handler = new LessConfiguredHandler(workers); + logger = Logger.getLogger(Feeder.class.getName()); + logLevel = logger.getLevel(); + logger.setLevel(Level.ALL); + initUseParentHandlers = logger.getUseParentHandlers(); + logChecker = new LogBuffer(); + logger.setUseParentHandlers(false); + logger.addHandler(logChecker); + } + + @After + public void tearDown() throws Exception { + handler.destroy(); + workers.shutdown(); + logger.setLevel(logLevel); + logger.removeHandler(logChecker); + logger.setUseParentHandlers(initUseParentHandlers); + } + + private static class LogBuffer extends Handler { + public final BlockingQueue<LogRecord> records = new LinkedBlockingQueue<>(); + + @Override + public void publish(LogRecord record) { + try { + records.put(record); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @Override + public void flush() { + } + + @Override + public void close() throws SecurityException { + } + } + + private static class LessConfiguredHandler extends FeedHandler { + volatile DataFormat lastFormatSeen; + + public LessConfiguredHandler(Executor executor) throws Exception { + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + } + + @Override + protected Feeder createFeeder(HttpRequest request, + InputStream requestInputStream, + BlockingQueue<OperationStatus> operations, + String clientId, + boolean sessionIdWasGeneratedJustNow, int protocolVersion) + throws Exception { + LessConfiguredFeeder f = new LessConfiguredFeeder(requestInputStream, operations, + popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, + sourceSessionParams(request), null, this, this.feedReplyHandler, "ourHostname"); + lastFormatSeen = f.settings.dataFormat; + return f; + } + + @Override + protected DocumentTypeManager createDocumentManager( + DocumentmanagerConfig documentManagerConfig) { + return null; + } + } + + private static class MockSharedSession extends SharedSourceSession { + + public MockSharedSession(SourceSessionParams params) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + ReplyHandler handler = msg.popHandler(); + MockReply mockReply = new MockReply(msg.getContext()); + if (msg instanceof Feeder.FeedErrorMessage) { + mockReply.addError(new com.yahoo.messagebus.Error(123, "Could not feed this")); + } + if (msg instanceof PutDocumentMessage) { + assert(msg.getTrace().getLevel() == 4); + assert(((PutDocumentMessage) msg).getPriority().name().equals("LOWEST")); + } + handler.handleReply(mockReply); + return Result.ACCEPTED; + } + + } + + private static class LessConfiguredFeeder extends Feeder { + public LessConfiguredFeeder(InputStream stream, + BlockingQueue<OperationStatus> operations, + ClientState storedState, FeederSettings settings, + String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, + SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, + String localHostname) throws Exception { + super(stream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, + sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); + } + + protected ReferencedResource<SharedSourceSession> retainSession( + SourceSessionParams sessionParams, SessionCache sessionCache) { + final SharedSourceSession session = new MockSharedSession(sessionParams); + return new ReferencedResource<>(session, References.fromResource(session)); + } + } + + @Test + public final void test() throws IOException, InterruptedException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); + assertEquals(7, logChecker.records.size()); + String actualHandshake = logChecker.records.take().getMessage(); + assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true)); + assertEquals("Successfully deserialized document id: id:banana:banana::doc1", + logChecker.records.take().getMessage()); + assertEquals("Sent message successfully, document id: id:banana:banana::doc1", + logChecker.records.take().getMessage()); + } + + //test session ID without #, i.e. something fishy related to VIPs is going on + sessionId = "something"; + + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "2"); + + HttpResponse r = handler.handle(nalle); + r.render(out); + String expectedErrorMsg = "Got request from client with id 'something', but found no session for this client. " + + "Most probably this server is in VIP rotation, and a client session was rotated from one " + + "server to another. This must not happen. Configure VIP with persistence=enabled, or " + + "(preferably) do not use a VIP at all."; + assertEquals(expectedErrorMsg, + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); + } + + //test session ID with trailing # but no hostname + sessionId = "something#"; + + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + HttpResponse r = handler.handle(nalle); + r.render(out); + String expectedErrorMsg = "Got request from client with id 'something#', but found no session for this " + + "client. Possible session timeout due to inactivity, server restart or " + + "reconfig, or bad VIP usage."; + assertEquals(expectedErrorMsg, + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); + } + + //test session ID with trailing # and some unknown hostname at the end + sessionId = "something#thisHostnameDoesNotExistAnywhere"; + + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + HttpResponse r = handler.handle(nalle); + r.render(out); + String expectedErrorMsg = "Got request from client with id 'something#thisHostnameDoesNotExistAnywhere', " + + "but found no session for this client. Session was originally established " + + "towards host thisHostnameDoesNotExistAnywhere, but our hostname is " + + "ourHostname. Most probably this server is in VIP rotation, and a session " + + "was rotated from one server to another. This should not happen. Configure VIP " + + "with persistence=enabled, or (preferably) do not use a VIP at all."; + assertEquals(expectedErrorMsg, + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.UTF_8.name(), r.getCharacterEncoding()); + } + + //test session ID with trailing # and some unknown hostname at the end + sessionId = "something#ourHostname"; + + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); + Thread.sleep(1000); + } + } + + @Test + public final void testFailedReading() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 4 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n", + Utf8.toString(out.toByteArray())); + } + } + + @Test + public final void testCleaningDoesNotBlowUp() throws IOException { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + handler.forceRunCleanClients(); + } + + @Test + public final void testMockNetworkDoesNotBlowUp() { + Network n = new MockNetwork(); + n.registerSession(null); + n.unregisterSession(null); + assertTrue(n.allocServiceAddress(null)); + n.freeServiceAddress(null); + n.send(null, null); + assertNull(n.getConnectionSpec()); + assertNull(n.getMirror()); + } + + @Test + public final void testMockReplyDoesNotBlowUp() { + MockReply r = new MockReply(null); + assertNull(r.getProtocol()); + assertEquals(0, r.getType()); + assertFalse(r.hasFatalErrors()); + } + + @Test + public final void testFlush() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1, 1, 1, 1, 1, 1, 1}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + } + } + + @Test + public final void testIllegalVersion() throws IOException { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers() + .add(Headers.VERSION, Integer.toString(Integer.MAX_VALUE)); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals(Headers.HTTP_NOT_ACCEPTABLE, r.getStatus()); + } + + @Test + public final void testSettings() { + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + nalle.getJDiscRequest().headers().add(Headers.ROUTE, "bamse brakar"); + nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); + FeederSettings settings = new FeederSettings(nalle); + assertEquals(false, settings.drain); + assertEquals(2, settings.route.getNumHops()); + assertEquals(true, settings.denyIfBusy); + } + + @Test + public final void testJsonInputFormat() throws IOException, InterruptedException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[]{1, 3, 2}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.TIMEOUT, "1000000000"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DATA_FORMAT, DataFormat.JSON_UTF8.name()); + nalle.getJDiscRequest().headers().add(Headers.PRIORITY, "LOWEST"); + nalle.getJDiscRequest().headers().add(Headers.TRACE_LEVEL, "4"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n" + + "id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + assertEquals("text/plain", r.getContentType()); + assertEquals(StandardCharsets.US_ASCII.name(), r.getCharacterEncoding()); + assertEquals(7, logChecker.records.size()); + String actualHandshake = logChecker.records.take().getMessage(); + assertThat(actualHandshake, actualHandshake.matches("Handshake completed for client (-?)(.+?)-#(.*?)\\."), is(true)); + assertEquals("Successfully deserialized document id: id:banana:banana::doc1", + logChecker.records.take().getMessage()); + assertEquals("Sent message successfully, document id: id:banana:banana::doc1", + logChecker.records.take().getMessage()); + assertSame(DataFormat.JSON_UTF8, handler.lastFormatSeen); + } + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java new file mode 100644 index 00000000000..4e6ca17abef --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2FailingMessagebusTestCase.java @@ -0,0 +1,224 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespaxmlparser.MockFeedReaderFactory; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; + +/** + * Check FeedHandler APIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class V2FailingMessagebusTestCase { + + LessConfiguredHandler handler; + ExecutorService workers; + int mbus; + + @Before + public void setUp() throws Exception { + workers = Executors.newCachedThreadPool(); + handler = new LessConfiguredHandler(workers); + mbus = 0; + } + + @After + public void tearDown() throws Exception { + handler.destroy(); + workers.shutdown(); + mbus = 0; + } + + private class LessConfiguredHandler extends FeedHandler { + + public LessConfiguredHandler(Executor executor) throws Exception { + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + } + + @Override + protected Feeder createFeeder(HttpRequest request, + InputStream requestInputStream, + BlockingQueue<OperationStatus> operations, + String clientId, + boolean sessionIdWasGeneratedJustNow, int protocolVersion) throws Exception { + return new LessConfiguredFeeder(requestInputStream, operations, + popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, + sourceSessionParams(request), null, this, this.feedReplyHandler, ""); + } + + @Override + protected DocumentTypeManager createDocumentManager( + DocumentmanagerConfig documentManagerConfig) { + return null; + } + } + + private class MockSharedSession extends SharedSourceSession { + + public MockSharedSession(SourceSessionParams params) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + ReplyHandler handler = msg.popHandler(); + + switch (mbus) { + case 0: + throw new RuntimeException("boom"); + case 1: + Result r = new Result(ErrorCode.SEND_QUEUE_FULL, "tralala"); + mbus = 2; + return r; + case 2: + handler.handleReply(new MockReply(msg.getContext())); + return Result.ACCEPTED; + default: + throw new IllegalStateException("WTF?!"); + } + } + } + + private class LessConfiguredFeeder extends Feeder { + + public LessConfiguredFeeder(InputStream inputStream, + BlockingQueue<OperationStatus> operations, + ClientState storedState, FeederSettings settings, + String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, + SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, + String localHostname) throws Exception { + super(inputStream, new MockFeedReaderFactory(), null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, + sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); + } + + protected ReferencedResource<SharedSourceSession> retainSession( + SourceSessionParams sessionParams, SessionCache sessionCache) { + final SharedSourceSession session = new MockSharedSession(sessionParams); + return new ReferencedResource<>(session, References.fromResource(session)); + } + } + + @Test + public final void testFailingMbus() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[]{1}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[]{1}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 ERROR boom \n", + Utf8.toString(out.toByteArray())); + } + } + + @Test + public final void testBusyMbus() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[]{1}); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + mbus = 2; + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + mbus = 1; + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers() + .add(Headers.DENY_IF_BUSY, "false"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 OK Document{20}processed. \n", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + mbus = 1; + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + nalle.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + assertEquals("id:banana:banana::doc1 TRANSIENT_ERROR tralala \n", + Utf8.toString(out.toByteArray())); + } + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java new file mode 100644 index 00000000000..1b2d855bdf2 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V2NoXmlReaderTestCase.java @@ -0,0 +1,162 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.messagebus.SessionCache; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.messagebus.*; +import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespa.http.client.core.OperationStatus; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.junit.Assert.assertEquals; + +/** + * Check FeedHandler APIs. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class V2NoXmlReaderTestCase { + + LessConfiguredHandler handler; + ExecutorService workers; + + @Before + public void setUp() throws Exception { + workers = Executors.newCachedThreadPool(); + handler = new LessConfiguredHandler(workers); + } + + @After + public void tearDown() throws Exception { + handler.destroy(); + workers.shutdown(); + } + + private static class LessConfiguredHandler extends FeedHandler { + + public LessConfiguredHandler(Executor executor) throws Exception { + super(executor, null, null, new DummyMetric(), AccessLog.voidAccessLog(), null); + } + + + @Override + protected Feeder createFeeder(HttpRequest request, InputStream requestInputStream, + BlockingQueue<OperationStatus> operations, String clientId, + boolean sessionIdWasGeneratedJustNow, int protocolVersion) + throws Exception { + return new LessConfiguredFeeder(requestInputStream, operations, + popClient(clientId), new FeederSettings(request), clientId, sessionIdWasGeneratedJustNow, + sourceSessionParams(request), null, this, this.feedReplyHandler, ""); + } + + @Override + protected DocumentTypeManager createDocumentManager( + DocumentmanagerConfig documentManagerConfig) { + return null; + } + } + + private static class MockSharedSession extends SharedSourceSession { + + public MockSharedSession(SourceSessionParams params) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + ReplyHandler handler = msg.popHandler(); + MockReply mockReply = new MockReply(msg.getContext()); + if (msg instanceof Feeder.FeedErrorMessage) { + mockReply.addError(new Error(123, "Could not feed this")); + } + handler.handleReply(mockReply); + return Result.ACCEPTED; + } + + } + + private static class LessConfiguredFeeder extends Feeder { + + public LessConfiguredFeeder(InputStream inputStream, + BlockingQueue<OperationStatus> operations, + ClientState storedState, FeederSettings settings, + String clientId, boolean sessionIdWasGeneratedJustNow, SourceSessionParams sessionParams, + SessionCache sessionCache, FeedHandler handler, ReplyHandler feedReplyHandler, + String localHostname) throws Exception { + super(inputStream, null, null, operations, storedState, settings, clientId, sessionIdWasGeneratedJustNow, + sessionParams, sessionCache, handler, new DummyMetric(), feedReplyHandler, localHostname); + } + + protected ReferencedResource<SharedSourceSession> retainSession( + SourceSessionParams sessionParams, SessionCache sessionCache) { + final SharedSourceSession session = new MockSharedSession(sessionParams); + return new ReferencedResource<>(session, References.fromResource(session)); + } + } + + @Test + public final void test() throws IOException { + String sessionId; + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest + .createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "false"); + HttpResponse r = handler.handle(nalle); + sessionId = r.headers().getFirst(Headers.SESSION_ID); + r.render(out); + assertEquals("", + Utf8.toString(out.toByteArray())); + } + { + InputStream in = new MetaStream(new byte[] { 1 }); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpRequest nalle = HttpRequest.createTestRequest( + "http://test4-steinar:19020/reserved-for-internal-use/feedapi", + Method.POST, in); + nalle.getJDiscRequest().headers().add(Headers.VERSION, "2"); + nalle.getJDiscRequest().headers().add(Headers.SESSION_ID, sessionId); + nalle.getJDiscRequest().headers().add(Headers.DRAIN, "true"); + HttpResponse r = handler.handle(nalle); + r.render(out); + //This is different from v1. If we cannot parse XML, we will still get response code 200, but with a sensible + //error message in the response. + assertEquals(200, r.getStatus()); + assertEquals("id:banana:banana::doc1 ERROR Could{20}not{20}feed{20}this \n", + Utf8.toString(out.toByteArray())); + } + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java new file mode 100644 index 00000000000..73095a3efe9 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.jdisc.Metric; +import com.yahoo.jdisc.ReferencedResource; +import com.yahoo.jdisc.References; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.Result; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.shared.SharedMessageBus; +import com.yahoo.messagebus.shared.SharedSourceSession; +import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespaxmlparser.MockFeedReaderFactory; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader; + +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; + + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; + + +public class V3CongestionTestCase { + AtomicInteger threadsAvail = new AtomicInteger(10); + AtomicInteger requests = new AtomicInteger(0); + + + static class ClientFeederWithMocks extends ClientFeederV3 { + + private final DocumentOperationMessageV3 docOp; + + ClientFeederWithMocks(ReferencedResource<SharedSourceSession> sourceSession, FeedReaderFactory feedReaderFactory, DocumentTypeManager docTypeManager, String clientId, Metric metric, ReplyHandler feedReplyHandler, AtomicInteger threadsAvailableForFeeding) { + super(sourceSession, feedReaderFactory, docTypeManager, clientId, metric, feedReplyHandler, threadsAvailableForFeeding); + // The operation to return from the client feeder. + VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); + docOp = DocumentOperationMessageV3.newRemoveMessage(op, "operation id"); + + } + + @Override + protected DocumentOperationMessageV3 getNextMessage( + String operationId, InputStream requestInputStream, FeederSettings settings) throws Exception { + while (true) { + int data = requestInputStream.read(); + if (data == -1 || data == (char)'\n') { + break; + } + } + return docOp; + } + } + + final static int NUMBER_OF_QUEUE_FULL_RESPONSES = 5; + + ClientFeederV3 clientFeederV3; + HttpRequest request; + + @Before + public void setup() { + // Set up a request to be used from the tests. + InputStream in = new MetaStream(new byte[] { 1 }); + request = HttpRequest + .createTestRequest( + "http://foo.bar:19020/reserved-for-internal-use/feedapi", + com.yahoo.jdisc.http.HttpRequest.Method.POST, in); + request.getJDiscRequest().headers().add(Headers.VERSION, "3"); + request.getJDiscRequest().headers().add(Headers.CLIENT_ID, "clientId"); + + + // Create a mock that does not parse the message, only reads the rest of the line. Makes it easier + // to write tests. It uses a mock for message bus. + clientFeederV3 = new ClientFeederWithMocks( + retainMockSession(new SourceSessionParams(), requests), + new MockFeedReaderFactory(), + null /*DocTypeManager*/, + "clientID", + null/*metric*/, + new FeedReplyReader(null/*metric*/), + threadsAvail); + } + + // A mock for message bus that can simulate blocking requests. + private static class MockSharedSession extends SharedSourceSession { + boolean queuFull = true; + AtomicInteger requests; + + public MockSharedSession(SourceSessionParams params, AtomicInteger requests) { + super(new SharedMessageBus(new MessageBus(new MockNetwork(), + new MessageBusParams())), params); + this.requests = requests; + } + + @Override + public Result sendMessageBlocking(Message msg) throws InterruptedException { + return sendMessage(msg); + } + + @Override + public Result sendMessage(Message msg) { + ReplyHandler handler = msg.popHandler(); + if (queuFull) { + requests.incrementAndGet(); + // Disable queue full after some attempts + if (requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES) { + queuFull = false; + } + Result r = new Result(ErrorCode.SEND_QUEUE_FULL, "queue full"); + return r; + } + + handler.handleReply(new MockReply(msg.getContext())); + return Result.ACCEPTED; + } + } + + ReferencedResource<SharedSourceSession> retainMockSession( + SourceSessionParams sessionParams, + AtomicInteger requests) { + final SharedSourceSession session = new MockSharedSession(sessionParams, requests); + return new ReferencedResource<>(session, References.fromResource(session)); + } + + @Test + public void testRetriesWhenThreadsAvailable() throws IOException { + request.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); + threadsAvail.set(10); + + clientFeederV3.handleRequest(request); + assertTrue(requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES); + } + + @Test + public void testNoRetriesWhenNoThreadsAvailable() throws IOException { + request.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "true"); + threadsAvail.set(0); + + clientFeederV3.handleRequest(request); + assertTrue(requests.get() == 1); + } + + @Test + public void testRetriesWhenNoThreadsAvailableButNoDenyIfBusy() throws IOException { + request.getJDiscRequest().headers().add(Headers.DENY_IF_BUSY, "false"); + threadsAvail.set(0); + + clientFeederV3.handleRequest(request); + assertTrue(requests.get() == NUMBER_OF_QUEUE_FULL_RESPONSES); + } +}
\ No newline at end of file diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java new file mode 100644 index 00000000000..df1de5d75ed --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/VersionsTestCase.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server; + +import com.yahoo.collections.Tuple2; +import com.yahoo.container.jdisc.HttpResponse; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.7.0 + */ +public class VersionsTestCase { + + private static final List<String> EMPTY = Collections.emptyList(); + private static final List<String> ONE_TWO = Arrays.asList("1", "2"); + private static final List<String> TWO_THREE = Arrays.asList("3", "2"); + private static final List<String> ONE_NULL_TWO = Arrays.asList("1", null, "2"); + private static final List<String> ONE_COMMA_TWO = Collections.singletonList("1, 2"); + private static final List<String> ONE_EMPTY_TWO = Arrays.asList("1", "", "2"); + private static final List<String> TOO_LARGE_NUMBER = Collections.singletonList("1000000000"); + private static final List<String> TWO_TOO_LARGE_NUMBER = Arrays.asList("2", "1000000000"); + private static final List<String> TWO_COMMA_TOO_LARGE_NUMBER = Arrays.asList("2,1000000000"); + private static final List<String> GARBAGE = Collections.singletonList("garbage"); + + @Test + public void testEmpty() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(EMPTY); + assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertThat(v.second, is(-1)); + } + + @Test + public void testOneTwo() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_TWO); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testTwoThree() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_THREE); + assertThat(v.first, nullValue()); + assertThat(v.second, is(3)); + } + + @Test + public void testOneNullTwo() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_NULL_TWO); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testOneCommaTwo() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_COMMA_TWO); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testOneEmptyTwo() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(ONE_EMPTY_TWO); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testTooLarge() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TOO_LARGE_NUMBER); + assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertThat(v.second, is(-1)); + } + + @Test + public void testTwoTooLarge() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_TOO_LARGE_NUMBER); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testTwoCommaTooLarge() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(TWO_COMMA_TOO_LARGE_NUMBER); + assertThat(v.first, nullValue()); + assertThat(v.second, is(2)); + } + + @Test + public void testGarbage() throws Exception { + Tuple2<HttpResponse, Integer> v = FeedHandler.doCheckProtocolVersion(GARBAGE); + assertThat(v.first, instanceOf(ErrorHttpResponse.class)); + assertThat(v.second, is(-1)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java new file mode 100644 index 00000000000..b134b5c34cd --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/util/ByteLimitedInputStreamTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.http.server.util; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.23 + */ +public class ByteLimitedInputStreamTestCase { + + private static ByteLimitedInputStream create(byte[] source, int limit) { + if (limit > source.length) { + throw new IllegalArgumentException("Limit is greater than length of source buffer."); + } + InputStream wrappedStream = new ByteArrayInputStream(source); + return new ByteLimitedInputStream(wrappedStream, limit); + } + + @Test + public void requireThatBasicsWork() throws IOException { + ByteLimitedInputStream stream = create("abcdefghijklmnopqr".getBytes(StandardCharsets.US_ASCII), 9); + + assertThat(stream.available(), is(9)); + assertThat(stream.read(), is(97)); + assertThat(stream.available(), is(8)); + assertThat(stream.read(), is(98)); + assertThat(stream.available(), is(7)); + assertThat(stream.read(), is(99)); + assertThat(stream.available(), is(6)); + assertThat(stream.read(), is(100)); + assertThat(stream.available(), is(5)); + assertThat(stream.read(), is(101)); + assertThat(stream.available(), is(4)); + assertThat(stream.read(), is(102)); + assertThat(stream.available(), is(3)); + assertThat(stream.read(), is(103)); + assertThat(stream.available(), is(2)); + assertThat(stream.read(), is(104)); + assertThat(stream.available(), is(1)); + assertThat(stream.read(), is(105)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + assertThat(stream.read(), is(-1)); + assertThat(stream.available(), is(0)); + } + + @Test + public void requireThatChunkedReadWorks() throws IOException { + ByteLimitedInputStream stream = create("abcdefghijklmnopqr".getBytes(StandardCharsets.US_ASCII), 9); + + assertThat(stream.available(), is(9)); + byte[] toBuf = new byte[4]; + assertThat(stream.read(toBuf), is(4)); + assertThat(toBuf[0], is((byte) 97)); + assertThat(toBuf[1], is((byte) 98)); + assertThat(toBuf[2], is((byte) 99)); + assertThat(toBuf[3], is((byte) 100)); + assertThat(stream.available(), is(5)); + + assertThat(stream.read(toBuf), is(4)); + assertThat(toBuf[0], is((byte) 101)); + assertThat(toBuf[1], is((byte) 102)); + assertThat(toBuf[2], is((byte) 103)); + assertThat(toBuf[3], is((byte) 104)); + assertThat(stream.available(), is(1)); + + assertThat(stream.read(toBuf), is(1)); + assertThat(toBuf[0], is((byte) 105)); + assertThat(stream.available(), is(0)); + + assertThat(stream.read(toBuf), is(-1)); + assertThat(stream.available(), is(0)); + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java new file mode 100644 index 00000000000..c245490b1d7 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockFeedReaderFactory.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.vespa.http.client.config.FeedParams; +import com.yahoo.vespa.http.server.FeedReaderFactory; + +import java.io.InputStream; + +/** + * For creating MockReader of innput stream. + * @author dybdahl + */ +public class MockFeedReaderFactory extends FeedReaderFactory { + + @Override + public FeedReader createReader( + InputStream inputStream, + DocumentTypeManager docTypeManager, + FeedParams.DataFormat dataFormat) { + try { + return new MockReader(inputStream); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java new file mode 100644 index 00000000000..a4ac0f4fdaf --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.vespa.http.server.MetaStream; +import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; +import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; + +import java.io.InputStream; +import java.lang.reflect.Field; + +/** + * Mock for ExternalFeedTestCase which had to override package private methods. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class MockReader implements FeedReader { + + MetaStream stream; + boolean finished = false; + + public MockReader(InputStream stream) throws Exception { + this.stream = getMetaStream(stream); + } + + private static MetaStream getMetaStream(InputStream stream) { + if (stream instanceof MetaStream) { + return (MetaStream) stream; + } + if (!(stream instanceof ByteLimitedInputStream)) { + throw new IllegalStateException("Given unknown stream type."); + } + //Ooooooo this is so ugly + try { + ByteLimitedInputStream byteLimitedInputStream = (ByteLimitedInputStream) stream; + Field f = byteLimitedInputStream.getClass().getDeclaredField("wrappedStream"); //NoSuchFieldException + f.setAccessible(true); + return (MetaStream) f.get(byteLimitedInputStream); + } catch (Exception e) { + throw new IllegalStateException("Implementation of ByteLimitedInputStream has changed.", e); + } + } + + @Override + public void read(Operation operation) throws Exception { + if (finished) { + return; + } + + byte whatToDo = stream.getNextOperation(); + DocumentId id = new DocumentId("id:banana:banana::doc1"); + DocumentType docType = new DocumentType("banana"); + switch (whatToDo) { + case 0: + finished = true; + break; + case 1: + Document doc = new Document(docType, id); + operation.setDocument(doc); + break; + case 2: + operation.setRemove(id); + break; + case 3: + operation.setDocumentUpdate(new DocumentUpdate(docType, id)); + break; + case 4: + throw new RuntimeException("boom"); + } + } + +}
\ No newline at end of file |