From 72231250ed81e10d66bfe70701e64fa5fe50f712 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Wed, 15 Jun 2016 23:09:44 +0200 Subject: Publish --- .../yahoo/search/federation/FutureWaiterTest.java | 109 ++++++ .../http/GzipDecompressingEntityTestCase.java | 212 +++++++++++ .../search/federation/http/HttpParametersTest.java | 238 ++++++++++++ .../search/federation/http/HttpPostTestCase.java | 99 +++++ .../yahoo/search/federation/http/HttpTestCase.java | 117 ++++++ .../yahoo/search/federation/http/PingTestCase.java | 278 ++++++++++++++ .../federation/http/QueryParametersTestCase.java | 65 ++++ .../com/yahoo/search/federation/image/.gitignore | 0 .../test/SearchChainResolverTestCase.java | 152 ++++++++ .../sourceref/test/SourceRefResolverTestCase.java | 114 ++++++ .../test/AddHitsWithRelevanceSearcher.java | 37 ++ .../search/federation/test/BlockingSearcher.java | 22 ++ .../federation/test/FederationSearcherTest.java | 306 +++++++++++++++ .../test/FederationSearcherTestCase.java | 411 +++++++++++++++++++++ .../search/federation/test/FederationTester.java | 75 ++++ .../search/federation/test/HitCountTestCase.java | 135 +++++++ .../federation/test/SetHitCountsSearcher.java | 39 ++ .../vespa/test/QueryMarshallerTestCase.java | 160 ++++++++ .../vespa/test/QueryParametersTestCase.java | 40 ++ .../vespa/test/ResultBuilderTestCase.java | 91 +++++ .../vespa/test/VespaIntegrationTestCase.java | 25 ++ .../vespa/test/VespaSearcherTestCase.java | 229 ++++++++++++ .../yahoo/search/federation/vespa/test/idhits.xml | 23 ++ .../search/federation/vespa/test/nestedhits.xml | 318 ++++++++++++++++ .../com/yahoo/search/federation/ysm/.gitignore | 0 25 files changed, 3295 insertions(+) create mode 100644 container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/image/.gitignore create mode 100644 container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml create mode 100644 container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml create mode 100644 container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore (limited to 'container-search/src/test/java/com/yahoo/search/federation') diff --git a/container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java b/container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java new file mode 100644 index 00000000000..37969e12399 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation; + + +/** + * @author tonytv + */ +// TODO: Fix or remove! +public class FutureWaiterTest { + +/* + + @MockClass(realClass = System.class) + public static class MockSystem { + + private static long currentTime; + private static boolean firstTime; + + private static final long startTime = 123; + + @Mock + public static synchronized long currentTimeMillis() { + if (firstTime) { + firstTime = false; + return startTime; + } + return currentTime; + } + + static synchronized void setElapsedTime(long elapsedTime) { + firstTime = true; + currentTime = elapsedTime + startTime; + } + } + + @Mocked() + FutureResult result1; + + @Mocked() + FutureResult result2; + + @Mocked() + FutureResult result3; + + @Mocked() + FutureResult result4; + + @Before + public void before() { + Mockit.setUpMock(FutureWaiterTest.MockSystem.class); + } + + @After + public void after() { + Mockit.tearDownMocks(); + } + + @Test + public void require_time_to_wait_is_adjusted_for_elapsed_time() { + MockSystem.setElapsedTime(300); + + FutureWaiter futureWaiter = new FutureWaiter(); + futureWaiter.add(result1, 350); + futureWaiter.waitForFutures(); + + new FullVerifications() { + { + result1.get(350 - 300, TimeUnit.MILLISECONDS); + } + }; + } + + @Test + public void require_do_not_wait_for_expired_timeouts() { + MockSystem.setElapsedTime(300); + + FutureWaiter futureWaiter = new FutureWaiter(); + futureWaiter.add(result1, 300); + futureWaiter.add(result2, 290); + + futureWaiter.waitForFutures(); + + new FullVerifications() { + {} + }; + } + + @Test + public void require_wait_for_largest_timeout_first() throws InterruptedException { + MockSystem.setElapsedTime(600); + + FutureWaiter futureWaiter = new FutureWaiter(); + futureWaiter.add(result1, 500); + futureWaiter.add(result4, 800); + futureWaiter.add(result2, 600); + futureWaiter.add(result3, 700); + + futureWaiter.waitForFutures(); + + new FullVerifications() { + { + result4.get(800 - 600, TimeUnit.MILLISECONDS); + result3.get(700 - 600, TimeUnit.MILLISECONDS); + } + }; + } + + */ +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java new file mode 100644 index 00000000000..c707702a3d3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java @@ -0,0 +1,212 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import static org.junit.Assert.*; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Random; +import java.util.zip.GZIPOutputStream; + +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.message.BasicHeader; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.text.Utf8; + +/** + * Test GZip support for the HTTP integration introduced in 4.2. + * + * @author Steinar Knutsen + */ +public class GzipDecompressingEntityTestCase { + private static final String STREAM_CONTENT = "00000000000000000000000000000000000000000000000000"; + private static final byte[] CONTENT_AS_BYTES = Utf8.toBytes(STREAM_CONTENT); + GzipDecompressingEntity testEntity; + + private static final class MockEntity implements HttpEntity { + + private final InputStream inStream; + + MockEntity(InputStream is) { + inStream = is; + } + + @Override + public boolean isRepeatable() { + return false; + } + + @Override + public boolean isChunked() { + return false; + } + + @Override + public long getContentLength() { + return -1; + } + + @Override + public Header getContentType() { + return new BasicHeader("Content-Type", "text/plain"); + } + + @Override + public Header getContentEncoding() { + return new BasicHeader("Content-Encoding", "gzip"); + } + + @Override + public InputStream getContent() throws IOException, + IllegalStateException { + return inStream; + } + + @Override + public void writeTo(OutputStream outstream) throws IOException { + } + + @Override + public boolean isStreaming() { + return false; + } + + @Override + public void consumeContent() throws IOException { + } + } + + @Before + public void setUp() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(out); + gzip.write(CONTENT_AS_BYTES); + gzip.finish(); + gzip.close(); + byte[] compressed = out.toByteArray(); + InputStream inStream = new ByteArrayInputStream(compressed); + testEntity = new GzipDecompressingEntity(new MockEntity(inStream)); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetContentLength() throws UnknownHostException { + assertEquals(STREAM_CONTENT.length(), testEntity.getContentLength()); + } + + @Test + public final void testGetContent() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length]; + int read = in.read(buffer); + assertEquals(CONTENT_AS_BYTES.length, read); + assertArrayEquals(CONTENT_AS_BYTES, buffer); + } + + @Test + public final void testGetContentToBigArray() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; + in.read(buffer); + byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); + assertArrayEquals(expected, buffer); + } + + @Test + public final void testGetContentAvailable() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + assertEquals(CONTENT_AS_BYTES.length, in.available()); + } + + @Test + public final void testLargeZip() throws IOException { + byte [] input = new byte [10000000]; + Random random = new Random(89); + for (int i = 0; i < input.length; i++) { + input[i] = (byte) random.nextInt(); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(out); + gzip.write(input); + gzip.finish(); + gzip.close(); + byte[] compressed = out.toByteArray(); + assertEquals(10003073, compressed.length); + InputStream inStream = new ByteArrayInputStream(compressed); + GzipDecompressingEntity gunzipper = new GzipDecompressingEntity(new MockEntity(inStream)); + assertEquals(input.length, gunzipper.getContentLength()); + byte[] buffer = new byte[input.length]; + InputStream content = gunzipper.getContent(); + assertEquals(input.length, content.available()); + int read = content.read(buffer); + assertEquals(input.length, read); + assertArrayEquals(input, buffer); + } + + @Test + public final void testGetContentReadByte() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; + int i = 0; + while (i < buffer.length) { + int r = in.read(); + if (r == -1) { + break; + } else { + buffer[i++] = (byte) r; + } + } + byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); + assertEquals(CONTENT_AS_BYTES.length, i); + assertArrayEquals(expected, buffer); + } + + @Test + public final void testGetContentReadWithOffset() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; + int read = in.read(buffer, CONTENT_AS_BYTES.length, CONTENT_AS_BYTES.length); + assertEquals(CONTENT_AS_BYTES.length, read); + byte[] expected = new byte[CONTENT_AS_BYTES.length * 2]; + for (int i = 0; i < CONTENT_AS_BYTES.length; ++i) { + expected[CONTENT_AS_BYTES.length + i] = CONTENT_AS_BYTES[i]; + } + assertArrayEquals(expected, buffer); + read = in.read(buffer, 0, CONTENT_AS_BYTES.length); + assertEquals(-1, read); + } + + @Test + public final void testGetContentSkip() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + final long n = 5L; + long skipped = in.skip(n); + assertEquals(n, skipped); + int read = in.read(); + assertEquals(CONTENT_AS_BYTES[(int) n], read); + skipped = in.skip(5000); + assertEquals(CONTENT_AS_BYTES.length - n - 1, skipped); + assertEquals(-1L, in.skip(1L)); + } + + + @Test + public final void testWriteToOutputStream() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + testEntity.writeTo(out); + assertArrayEquals(CONTENT_AS_BYTES, out.toByteArray()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java new file mode 100644 index 00000000000..c3bd2ada260 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java @@ -0,0 +1,238 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.search.federation.ProviderConfig; +import org.junit.Test; + +import static com.yahoo.search.federation.ProviderConfig.Yca; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + * @author Steinar Knutsen + */ +public class HttpParametersTest { + + @Test + public void create_from_config() throws Exception { + ProviderConfig config = new ProviderConfig(new ProviderConfig.Builder() + .connectionTimeout(1.0) + .maxConnectionPerRoute(2) + .maxConnections(3) + .path("myPath") + .readTimeout(4) + .socketBufferBytes(5) + .yca(new Yca.Builder() + .applicationId("myId") + .host("myYcaHost") + .port(7) + .retry(8) + .ttl(9) + .useProxy(true))); + + HTTPParameters httpParameters = new HTTPParameters(config); + + // Written to configuredConnectionTimeout, but it is not accessible!? + //assertThat(httpParameters.getConnectionTimeout(), is(1000)); + + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getMaxConnectionsPerRoute(), is(2)); + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getMaxTotalConnections(), is(3)); + + assertThat(httpParameters.getPath(), is("/myPath")); + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getReadTimeout(), is(4)); + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getSocketBufferSizeBytes(), is(5)); + + + assertThat(httpParameters.getYcaUseProxy(), is(true)); + assertThat(httpParameters.getYcaApplicationId(), is("myId")); + assertThat(httpParameters.getYcaProxy(), is("myYcaHost")); + assertThat(httpParameters.getYcaPort(), is(7)); + assertThat(httpParameters.getYcaRetry(), is(8000L)); + assertThat(httpParameters.getYcaTtl(), is(9000L)); + } + + @Test + public void requireFreezeWorksForAccessors() { + HTTPParameters p = new HTTPParameters(); + boolean caught = false; + final int expected = 37; + p.setConnectionTimeout(expected); + assertEquals(expected, p.getConnectionTimeout()); + p.freeze(); + try { + p.setConnectionTimeout(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setReadTimeout(expected); + assertEquals(expected, p.getReadTimeout()); + p.freeze(); + try { + p.setReadTimeout(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setPersistentConnections(true); + assertTrue(p.getPersistentConnections()); + p.freeze(); + try { + p.setPersistentConnections(false); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + assertEquals("http", p.getProxyType()); + + p = new HTTPParameters(); + caught = false; + p.setEnableProxy(true); + assertTrue(p.getEnableProxy()); + p.freeze(); + try { + p.setEnableProxy(false); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setProxyHost("nalle"); + assertEquals("nalle", p.getProxyHost()); + p.freeze(); + try { + p.setProxyHost("jappe"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setProxyPort(expected); + assertEquals(expected, p.getProxyPort()); + p.freeze(); + try { + p.setProxyPort(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setMethod("POST"); + assertEquals("POST", p.getMethod()); + p.freeze(); + try { + p.setMethod("GET"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setSchema("gopher"); + assertEquals("gopher", p.getSchema()); + p.freeze(); + try { + p.setSchema("http"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setInputEncoding("iso-8859-15"); + assertEquals("iso-8859-15", p.getInputEncoding()); + p.freeze(); + try { + p.setInputEncoding("shift-jis"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setOutputEncoding("iso-8859-15"); + assertEquals("iso-8859-15", p.getOutputEncoding()); + p.freeze(); + try { + p.setOutputEncoding("shift-jis"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setMaxTotalConnections(expected); + assertEquals(expected, p.getMaxTotalConnections()); + p.freeze(); + try { + p.setMaxTotalConnections(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setMaxConnectionsPerRoute(expected); + assertEquals(expected, p.getMaxConnectionsPerRoute()); + p.freeze(); + try { + p.setMaxConnectionsPerRoute(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setSocketBufferSizeBytes(expected); + assertEquals(expected, p.getSocketBufferSizeBytes()); + p.freeze(); + try { + p.setSocketBufferSizeBytes(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setRetries(expected); + assertEquals(expected, p.getRetries()); + p.freeze(); + try { + p.setRetries(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java new file mode 100644 index 00000000000..8edc1ca8dd8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.StupidSingleThreadedHttpServer; +import com.yahoo.search.federation.ProviderConfig.PingOption; +import com.yahoo.search.federation.http.Connection; +import com.yahoo.search.federation.http.HTTPProviderSearcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.StringEntity; +import org.junit.Test; + +import java.io.InputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.core.StringContains.containsString; +import static org.junit.Assert.assertThat; + +/** + * See bug #3234696. + * + * @author Einar M R Rosenvinge + */ +public class HttpPostTestCase { + + @Test + public void testPostingSearcher() throws Exception { + StupidSingleThreadedHttpServer server = new StupidSingleThreadedHttpServer(); + server.start(); + + TestPostSearcher searcher = new TestPostSearcher(new ComponentId("foo:1"), + Arrays.asList(new Connection("localhost", server.getServerPort())), + "/"); + Query q = new Query(""); + q.setTimeout(10000000L); + Execution e = new Execution(searcher, Execution.Context.createContextStub()); + + searcher.search(q, e); + + assertThat(server.getRequest(), containsString("My POST body")); + server.stop(); + } + + private static class TestPostSearcher extends HTTPProviderSearcher { + public TestPostSearcher(ComponentId id, List connections, String path) { + super(id, connections, httpParameters(path), Statistics.nullImplementation); + } + + private static HTTPParameters httpParameters(String path) { + HTTPParameters httpParameters = new HTTPParameters(path); + httpParameters.setPingOption(PingOption.Enum.DISABLE); + return httpParameters; + } + + @Override + protected HttpUriRequest createRequest(String method, URI uri, HttpEntity entity) { + HttpPost request = new HttpPost(uri); + request.setEntity(entity); + return request; + } + + @Override + protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { + try { + return new StringEntity("My POST body"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + @Override + public Map getCacheKey(Query q) { + return new HashMap<>(0); + } + + @Override + public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { + // do nothing with the result + } + + @Override + protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { + //Empty + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java new file mode 100644 index 00000000000..c59dffb9cb7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.StupidSingleThreadedHttpServer; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import com.yahoo.text.Utf8; + +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Rudimentary http searcher test. + * + * @author Jon Bratseth + */ +public class HttpTestCase extends junit.framework.TestCase { + + private StupidSingleThreadedHttpServer httpServer; + private TestHTTPClientSearcher searcher; + + public void testSearcher() throws JAXBException { + Result result = searchUsingLocalhost(); + + assertEquals("ok", result.getQuery().properties().get("gotResponse")); + assertEquals(0, result.getQuery().errors().size()); + } + + private Result searchUsingLocalhost() { + searcher = new TestHTTPClientSearcher("test","localhost",getPort()); + Query query = new Query("/?query=test"); + + query.setWindow(0,10); + return searcher.search(query, new Execution(searcher, Execution.Context.createContextStub())); + } + + public void test_that_ip_address_set_on_meta_hit() { + Result result = searchUsingLocalhost(); + Hit metaHit = getFirstMetaHit(result.hits()); + String ip = (String) metaHit.getField(HTTPSearcher.LOG_IP_ADDRESS); + + assertEquals(ip, "127.0.0.1"); + } + + private Hit getFirstMetaHit(HitGroup hits) { + for (Iterator i = hits.unorderedDeepIterator(); i.hasNext();) { + Hit hit = i.next(); + if (hit.isMeta()) + return hit; + } + return null; + } + + @Override + public void setUp() throws Exception { + httpServer = new StupidSingleThreadedHttpServer(0, 0) { + @Override + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 5\r\n" + + "\r\n" + + "hello"); + } + }; + httpServer.start(); + } + + private int getPort() { + return httpServer.getServerPort(); + } + + @Override + public void tearDown() throws Exception { + httpServer.stop(); + if (searcher != null) { + searcher.shutdownConnectionManagers(); + } + } + + private static class TestHTTPClientSearcher extends HTTPClientSearcher { + + public TestHTTPClientSearcher(String id, String hostName, int port) { + super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); + } + + private static List toConnections(String hostName,int port) { + List connections=new ArrayList<>(); + connections.add(new Connection(hostName,port)); + return connections; + } + + @Override + public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { + query.properties().set("gotResponse","ok"); + return query; + } + + @Override + public Map getCacheKey(Query q) { + return null; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java new file mode 100644 index 00000000000..34791168db4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java @@ -0,0 +1,278 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.StupidSingleThreadedHttpServer; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import com.yahoo.text.Utf8; +import com.yahoo.yolean.Exceptions; +import org.apache.http.HttpEntity; + +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Check for different keep-alive scenarios. What we really want to test + * is the server does not hang. + * + * @author Steinar Knutsen + */ +public class PingTestCase extends junit.framework.TestCase { + static final int TIMEOUT_MS = 60000; + public void testNiceCase() throws Exception { + NiceStupidServer server = new NiceStupidServer(); + server.start(); + checkSearchAndPing(true, true, true, server.getServerPort()); + server.stop(); + } + + private void checkSearchAndPing(boolean firstSearch, boolean pongCheck, boolean secondSearch, int port) { + String resultThing; + String comment; + TestHTTPClientSearcher searcher = new TestHTTPClientSearcher("test", + "localhost", port); + try { + + Query query = new Query("/?query=test"); + + query.setWindow(0, 10); + // high timeout to allow for overloaded test machine + query.setTimeout(TIMEOUT_MS); + Ping ping = new Ping(TIMEOUT_MS); + + long start = System.currentTimeMillis(); + Execution exe = new Execution(searcher, Execution.Context.createContextStub()); + exe.search(query); + + resultThing = firstSearch ? "ok" : null; + comment = firstSearch ? "First search should have succeeded." : "First search should fail."; + assertEquals(comment, resultThing, query.properties().get("gotResponse")); + Pong pong = searcher.ping(ping, searcher.getConnection()); + if (pongCheck) { + assertEquals("Ping should not have failed.", 0, pong.getErrorSize()); + } else { + assertEquals("Ping should have failed.", 1, pong.getErrorSize()); + } + exe = new Execution(searcher, Execution.Context.createContextStub()); + exe.search(query); + + resultThing = secondSearch ? "ok" : null; + comment = secondSearch ? "Second search should have succeeded." : "Second search should fail."; + + assertEquals(resultThing, query.properties().get("gotResponse")); + long duration = System.currentTimeMillis() - start; + // target for duration based on the timeout values + some slack + assertTrue("This test probably hanged.", duration < TIMEOUT_MS + 4000); + searcher.shutdownConnectionManagers(); + } finally { + searcher.deconstruct(); + } + } + + public void testUselessCase() throws Exception { + UselessStupidServer server = new UselessStupidServer(); + server.start(); + checkSearchAndPing(false, true, false, server.getServerPort()); + server.stop(); + } + + public void testGrumpyCase() throws Exception { + GrumpyStupidServer server = new GrumpyStupidServer(); + server.start(); + checkSearchAndPing(false, false, false, server.getServerPort()); + server.stop(); + } + + public void testPassiveAggressiveCase() throws Exception { + PassiveAggressiveStupidServer server = new PassiveAggressiveStupidServer(); + server.start(); + checkSearchAndPing(true, false, true, server.getServerPort()); + server.stop(); + } + + // OK on ping and search + private static class NiceStupidServer extends StupidSingleThreadedHttpServer { + private NiceStupidServer() throws IOException { + super(0, 0); + } + + @Override + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "hello\n"); + } + } + + // rejects ping and accepts search + private static class PassiveAggressiveStupidServer extends StupidSingleThreadedHttpServer { + + private PassiveAggressiveStupidServer() throws IOException { + super(0, 0); + } + + @Override + protected byte[] getResponse(String request) { + if (request.contains("/ping")) { + return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 8\r\n" + + "\r\n" + + "go away\n"); + } else { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "hello\n"); + } + } + } + + // accepts ping and rejects search + private static class UselessStupidServer extends StupidSingleThreadedHttpServer { + private UselessStupidServer() throws IOException { + super(0, 0); + } + + + @Override + protected byte[] getResponse(String request) { + if (request.contains("/ping")) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "hello\n"); + } else { + return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 8\r\n" + + "\r\n" + + "go away\n"); + } + } + } + + // rejects ping and search + private static class GrumpyStupidServer extends StupidSingleThreadedHttpServer { + private GrumpyStupidServer() throws IOException { + super(0, 0); + } + + @Override + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 8\r\n" + + "\r\n" + + "go away\n"); + } + } + + private static class TestHTTPClientSearcher extends HTTPClientSearcher { + + public TestHTTPClientSearcher(String id, String hostName, int port) { + super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); + } + + private static List toConnections(String hostName,int port) { + List connections=new ArrayList<>(); + connections.add(new Connection(hostName,port)); + return connections; + } + + @Override + public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { + query.properties().set("gotResponse","ok"); + return query; + } + + @Override + public Result search(Query query, Execution execution, + Connection connection) { + URI uri; + try { + uri = new URL("http", connection.getHost(), connection + .getPort(), "/search").toURI(); + } catch (MalformedURLException e) { + query.errors().add(createMalformedUrlError(query, e)); + return execution.search(query); + } catch (URISyntaxException e) { + query.errors().add(createMalformedUrlError(query, e)); + return execution.search(query); + } + + HttpEntity entity; + try { + entity = getEntity(uri, query); + } catch (IOException e) { + query.errors().add( + ErrorMessage.createBackendCommunicationError("Error when trying to connect to HTTP backend in " + + this + " using " + connection + + " for " + query + ": " + + Exceptions.toMessageString(e))); + return execution.search(query); + } catch (TimeoutException e) { + query.errors().add(ErrorMessage.createTimeout("No time left for HTTP traffic in " + + this + + " for " + query + ": " + e.getMessage())); + return execution.search(query); + } + if (entity == null) { + query.errors().add( + ErrorMessage.createBackendCommunicationError("No result from connecting to HTTP backend in " + + this + " using " + connection + " for " + query)); + return execution.search(query); + } + + try { + query = handleResponse(entity, query); + } catch (IOException e) { + query.errors().add( + ErrorMessage.createBackendCommunicationError("Error when trying to consume input in " + + this + ": " + Exceptions.toMessageString(e))); + } finally { + cleanupHttpEntity(entity); + } + return execution.search(query); + } + + @Override + public Map getCacheKey(Query q) { + return null; + } + + @Override + protected URI getPingURI(Connection connection) + throws MalformedURLException, URISyntaxException { + return new URL("http", connection.getHost(), connection.getPort(), "/ping").toURI(); + } + + Connection getConnection() { + return getHasher().getNodes().select(0, 0); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java new file mode 100644 index 00000000000..baeb9fd0a41 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.federation.vespa.VespaSearcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import com.yahoo.vespa.defaults.Defaults; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Tests that source and backend specific parameters from the query are added correctly to the backend requests + * + * @author Jon Bratseth + */ +public class QueryParametersTestCase extends junit.framework.TestCase { + + public void testQueryParameters() { + Query query=new Query(); + query.properties().set("a","a-value"); + query.properties().set("b.c","b.c-value"); + query.properties().set("source.otherSource.d","d-value"); + query.properties().set("source.testSource.e","e-value"); + query.properties().set("source.testSource.f.g","f.g-value"); + query.properties().set("provider.testProvider.h","h-value"); + query.properties().set("provider.testProvider.i.j","i.j-value"); + + query.properties().set("sourceName","testSource"); // Done by federation searcher + query.properties().set("providerName","testProvider"); // Done by federation searcher + + TestHttpProvider searcher=new TestHttpProvider(); + Map parameters=searcher.getQueryMap(query); + searcher.deconstruct(); + + assertEquals(4,parameters.size()); // the appropriate 4 of the above + assertEquals(parameters.get("e"),"e-value"); + assertEquals(parameters.get("f.g"),"f.g-value"); + assertEquals(parameters.get("h"),"h-value"); + assertEquals(parameters.get("i.j"),"i.j-value"); + } + + public static class TestHttpProvider extends HTTPProviderSearcher { + + public TestHttpProvider() { + super(new ComponentId("test"), Collections.singletonList(new Connection("host", Defaults.getDefaults().vespaWebServicePort())), "path", Statistics.nullImplementation); + } + + @Override + public Map getCacheKey(Query q) { + return Collections.singletonMap("nocaching", String.valueOf(Math.random())); + } + + @Override + protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { + } + + } + +} + diff --git a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore new file mode 100644 index 00000000000..e69de29bb2d diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java new file mode 100644 index 00000000000..e874c89b918 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.processing.request.properties.PropertyMap; +import com.yahoo.processing.request.Properties; +import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.federation.sourceref.Target; +import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.junit.Test; + +import java.util.Collections; +import java.util.Iterator; +import java.util.SortedSet; + +import static junit.framework.Assert.assertNull; +import static junit.framework.Assert.fail; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class SearchChainResolverTestCase { + + private static final FederationOptions federationOptions = + new FederationOptions().setTimeoutInMilliseconds(3000).setOptional(true); + + private static final ComponentId searchChainId = ComponentId.fromString("search-chain"); + private static final ComponentId providerId = ComponentId.fromString("provider"); + private static final ComponentId provider2Id = ComponentId.fromString("provider2"); + + private static final ComponentId sourceId = ComponentId.fromString("source"); + private static final ComponentId sourceChainInProviderId = + ComponentId.fromString("source-chain").nestInNamespace(providerId); + private static final ComponentId sourceChainInProvider2Id = + ComponentId.fromString("source-chain").nestInNamespace(provider2Id); + + private static final SearchChainResolver searchChainResolver; + + static { + SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + builder.addSearchChain(searchChainId, federationOptions.setUseByDefault(true), Collections.emptyList()); + builder.addSearchChain(providerId, federationOptions.setUseByDefault(false), Collections.emptyList()); + builder.addSourceForProvider(sourceId, providerId, sourceChainInProviderId, true, + federationOptions.setUseByDefault(true), Collections.emptyList()); + builder.addSourceForProvider(sourceId, provider2Id, sourceChainInProvider2Id, false, + federationOptions.setUseByDefault(false), Collections.emptyList()); + + searchChainResolver = builder.build(); + } + + @Test + public void check_default_search_chains() { + assertThat(searchChainResolver.defaultTargets().size(), is(2)); + + Iterator iterator = searchChainResolver.defaultTargets().iterator(); + assertThat(iterator.next().searchRefDescription(), is(searchChainId.toString())); + assertThat(iterator.next().searchRefDescription(), is(sourceChainInProviderId.toString())); + } + + @Test + public void require_error_message_for_invalid_source() { + try { + resolve("no-such-source"); + fail("Expected exception."); + } catch (UnresolvedSearchChainException e) { + assertThat(e.getMessage(), is("Could not resolve source ref 'no-such-source'.")); + } + } + + @Test + public void lookup_search_chain() throws Exception { + SearchChainInvocationSpec res = resolve(searchChainId.getName()); + assertThat(res.searchChainId, is(searchChainId)); + } + + //TODO: TVT: @Test() + public void lookup_provider() throws Exception { + SearchChainInvocationSpec res = resolve(providerId.getName()); + assertThat(res.provider, is(providerId)); + assertNull(res.source); + assertThat(res.searchChainId, is(providerId)); + } + + @Test + public void lookup_source() throws Exception { + SearchChainInvocationSpec res = resolve(sourceId.getName()); + assertIsSourceInProvider(res); + } + + @Test + public void lookup_source_search_chain_directly() throws Exception { + SearchChainInvocationSpec res = resolve(sourceChainInProviderId.stringValue()); + assertIsSourceInProvider(res); + } + + private void assertIsSourceInProvider(SearchChainInvocationSpec res) { + assertThat(res.provider, is(providerId)); + assertThat(res.source, is(sourceId)); + assertThat(res.searchChainId, is(sourceChainInProviderId)); + } + + @Test + public void lookup_source_for_provider2() throws Exception { + SearchChainInvocationSpec res = resolve(sourceId.getName(), provider2Id.getName()); + assertThat(res.provider, is(provider2Id)); + assertThat(res.source, is(sourceId)); + assertThat(res.searchChainId, is(sourceChainInProvider2Id)); + } + + @Test + public void lists_source_ref_description_for_top_level_targets() { + SortedSet topLevelTargets = searchChainResolver.allTopLevelTargets(); + assertThat(topLevelTargets.size(), is(3)); + + Iterator i = topLevelTargets.iterator(); + assertSearchRefDescriptionIs(i.next(), providerId.toString()); + assertSearchRefDescriptionIs(i.next(), searchChainId.toString()); + assertSearchRefDescriptionIs(i.next(), "source[provider = provider, provider2]"); + } + + private void assertSearchRefDescriptionIs(Target target, String expected) { + assertThat(target.searchRefDescription(), is(expected)); + } + + static Properties emptySourceToProviderMap() { + return new PropertyMap(); + } + + private SearchChainInvocationSpec resolve(String sourceSpecification) throws UnresolvedSearchChainException { + return resolve(sourceSpecification, emptySourceToProviderMap()); + } + + private SearchChainInvocationSpec resolve(String sourceSpecification, String providerSpecification) + throws UnresolvedSearchChainException { + Properties sourceToProviderMap = emptySourceToProviderMap(); + sourceToProviderMap.set("source." + sourceSpecification + ".provider", providerSpecification); + return resolve(sourceSpecification, sourceToProviderMap); + } + + private SearchChainInvocationSpec resolve(String sourceSpecification, Properties sourceToProviderMap) + throws UnresolvedSearchChainException { + SearchChainInvocationSpec res = searchChainResolver.resolve( + ComponentSpecification.fromString(sourceSpecification), sourceToProviderMap); + assertThat(res.federationOptions, is(federationOptions)); + return res; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java new file mode 100644 index 00000000000..f8559745358 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.federation.sourceref.SourceRefResolver; +import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeMap; + +import static com.yahoo.search.federation.sourceref.test.SearchChainResolverTestCase.emptySourceToProviderMap; +import static junit.framework.Assert.fail; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.matchers.JUnitMatchers.hasItems; + + +/** + * Test for SourceRefResolver. + * @author tonytv + */ +public class SourceRefResolverTestCase { + private static final String cluster1 = "cluster1"; + private static final String cluster2 = "cluster2"; + private static final String cluster3 = "cluster3"; + private static IndexFacts indexFacts; + + private static final SourceRefResolver sourceRefResolver = createSourceRefResolver(); + + static { + setupIndexFacts(); + } + + private static SourceRefResolver createSourceRefResolver() { + SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + builder.addSearchChain(ComponentId.fromString(cluster1), new FederationOptions().setUseByDefault(true), + Collections.emptyList()); + builder.addSearchChain(ComponentId.fromString(cluster2), new FederationOptions().setUseByDefault(true), + Collections.emptyList()); + + return new SourceRefResolver(builder.build()); + } + + private static void setupIndexFacts() { + TreeMap> masterClusters = new TreeMap<>(); + masterClusters.put(cluster1, Arrays.asList("document1", "document2")); + masterClusters.put(cluster2, Arrays.asList("document1")); + masterClusters.put(cluster3, Arrays.asList("document3")); + indexFacts = new IndexFacts(new IndexModel(masterClusters, null, null)); + } + + @Test + public void check_test_assumptions() { + assertThat(indexFacts.clustersHavingSearchDefinition("document1"), hasItems("cluster1", "cluster2")); + } + + @Test + public void lookup_search_chain() throws Exception { + Set searchChains = resolve(cluster1); + assertThat(searchChains.size(), is(1)); + assertThat(searchChainIds(searchChains), hasItems(cluster1)); + } + + @Test + public void lookup_search_chains_for_document1() throws Exception { + Set searchChains = resolve("document1"); + assertThat(searchChains.size(), is(2)); + assertThat(searchChainIds(searchChains), hasItems(cluster1, cluster2)); + } + + @Test + public void error_when_document_gives_cluster_without_matching_search_chain() { + try { + resolve("document3"); + fail("Expected exception"); + } catch (UnresolvedSearchChainException e) { + assertThat(e.getMessage(), is("Failed to resolve cluster search chain 'cluster3' " + + "when using source ref 'document3' as a document name.")); + } + } + + @Test + public void error_when_no_document_or_search_chain() { + try { + resolve("document4"); + fail("Expected exception"); + } catch (UnresolvedSearchChainException e) { + assertThat(e.getMessage(), is("Could not resolve source ref 'document4'.")); + } + } + + private List searchChainIds(Set searchChains) { + List names = new ArrayList<>(); + for (SearchChainInvocationSpec searchChain : searchChains) { + names.add(searchChain.searchChainId.stringValue()); + } + return names; + } + + private Set resolve(String documentName) throws UnresolvedSearchChainException { + return sourceRefResolver.resolve(ComponentSpecification.fromString(documentName), emptySourceToProviderMap(), indexFacts); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java b/container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java new file mode 100644 index 00000000000..40786ee89a9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author tonytv + */ +public class AddHitsWithRelevanceSearcher extends Searcher { + public static final int numHitsAdded = 5; + + private final String chainName; + private final int relevanceMultiplier; + + public AddHitsWithRelevanceSearcher(String chainName, int rankMultiplier) { + this.chainName = chainName; + this.relevanceMultiplier = rankMultiplier; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + for (int i = 1; i <= numHitsAdded; ++i) { + result.hits().add(createHit(i)); + } + return result; + } + + private Hit createHit(int i) { + int relevance = i * relevanceMultiplier; + return new Hit(chainName + "-" + relevance, relevance); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java b/container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java new file mode 100644 index 00000000000..dcecf36f2ae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author tonytv + */ +public class BlockingSearcher extends Searcher { + @Override + public synchronized Result search(Query query, Execution execution) { + try { + while (true) + wait(); + } catch (InterruptedException e) { + } + return execution.search(query); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java new file mode 100644 index 00000000000..dba0deb607a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java @@ -0,0 +1,306 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import java.util.Optional; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.net.URI; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.execution.chain.ChainRegistry; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.selection.FederationTarget; +import com.yahoo.search.federation.selection.TargetSelector; +import com.yahoo.search.federation.StrictContractsConfig; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.Execution.Context; +import com.yahoo.search.searchchain.model.federation.FederationOptions; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author tonytv + */ +public class FederationSearcherTest { + private static final String hasBeenFilled = "hasBeenFilled"; + + private static class AddHitSearcher extends Searcher { + protected Hit hit = createHit(); + + private Hit createHit() { + Hit hit = new Hit("dummy"); + hit.setFillable(); + return hit; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(hit); + return result; + } + + @Override + public void fill(Result result, String summaryClass, Execution execution) { + if (firstHit(result) != hit) { + throw new RuntimeException("Unknown hit"); + } + firstHit(result).setField(hasBeenFilled, true); + } + } + + private static class ModifyQueryAndAddHitSearcher extends AddHitSearcher { + private final String marker; + + ModifyQueryAndAddHitSearcher(String marker) { + super(); + this.marker = marker; + } + + @Override + public Result search(Query query, Execution execution) { + query.getModel().getQueryTree().setRoot(new WordItem(marker)); + Result result = execution.search(query); + result.hits().add(hit); + return result; + } + + } + + @Test + public void require_that_hits_are_not_automatically_filled() { + Result result = federationToSingleAddHitSearcher().search(); + assertNotFilled(firstHitInFirstGroup(result)); + } + + @Test + public void require_that_hits_can_be_filled() { + Result result = federationToSingleAddHitSearcher().searchAndFill(); + assertFilled(firstHitInFirstGroup(result)); + } + + @Test + public void require_that_hits_can_be_filled_when_moved() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + tester.addSearchChain("chain2", new AddHitSearcher()); + + Result result = tester.search(); + + Result reorganizedResult = new Result(result.getQuery()); + HitGroup hit1 = new HitGroup(); + HitGroup nestedHitGroup = new HitGroup(); + + hit1.add(nestedHitGroup); + reorganizedResult.hits().add(hit1); + + HitGroup chain1Group = (HitGroup) result.hits().get(0); + HitGroup chain2Group = (HitGroup) result.hits().get(1); + + nestedHitGroup.add(chain1Group.get(0)); + reorganizedResult.hits().add(chain2Group.get(0)); + reorganizedResult.hits().add(nestedHitGroup); + + tester.fill(reorganizedResult); + assertFilled(nestedHitGroup.get(0)); + assertFilled(chain2Group.get(0)); + + } + + @Test + public void require_that_hits_can_be_filled_for_multiple_chains_and_queries() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + tester.addSearchChain("chain2", new ModifyQueryAndAddHitSearcher("modified1")); + tester.addSearchChain("chain3", new ModifyQueryAndAddHitSearcher("modified2")); + + Result result = tester.search(); + tester.fill(result); + for (Iterator i = result.hits().deepIterator(); i.hasNext();) { + Hit h = i.next(); + assertFilled(h); + } + assertEquals(3, result.hits().getConcreteSize()); + } + + + @Test + public void require_that_optional_search_chains_does_not_delay_federation() { + BlockingSearcher blockingSearcher = new BlockingSearcher(); + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + tester.addOptionalSearchChain("chain2", blockingSearcher); + + Result result = tester.searchAndFill(); + assertThat(getNonErrorHits(result).size(), is(1)); + assertFilled(getFirstHit(getNonErrorHits(result).get(0))); + assertNotNull(result.hits().getError()); + } + + @Test + public void require_that_calling_a_single_slow_source_with_long_timeout_does_not_delay_federation() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", + new FederationOptions().setUseByDefault(true).setRequestTimeoutInMilliseconds(3600 * 1000), + new BlockingSearcher() ); + + Query query = new Query(); + query.setTimeout(2); // make the test run faster + Result result = tester.search(query); + assertThat(getNonErrorHits(result).size(), is(0)); + assertNotNull(result.hits().getError()); + } + + private Hit getFirstHit(Hit hitGroup) { + if (hitGroup instanceof HitGroup) + return ((HitGroup) hitGroup).get(0); + else + throw new IllegalArgumentException("Expected HitGroup"); + } + + private List getNonErrorHits(Result result) { + List nonErrorHits = new ArrayList<>(); + for (Hit hit : result.hits()) { + if (!(hit instanceof ErrorHit)) + nonErrorHits.add(hit); + } + + return nonErrorHits; + } + private static void assertFilled(Hit hit) { + assertTrue((Boolean)hit.getField(hasBeenFilled)); + } + + private static void assertNotFilled(Hit hit) { + assertNull(hit.getField(hasBeenFilled)); + } + + private FederationTester federationToSingleAddHitSearcher() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + return tester; + } + + private static Hit firstHit(Result result) { + return result.hits().get(0); + } + + private static Hit firstHitInFirstGroup(Result result) { + return ((HitGroup)firstHit(result)).get(0); + } + + @Test + public void custom_federation_target() { + ComponentId targetSelectorId = ComponentId.fromString("TargetSelector"); + ComponentRegistry targetSelectors = new ComponentRegistry<>(); + targetSelectors.register(targetSelectorId, new TestTargetSelector()); + + FederationSearcher searcher = new FederationSearcher( + new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), + new StrictContractsConfig(new StrictContractsConfig.Builder()), + targetSelectors); + + Result result = new Execution(searcher, Context.createContextStub()).search(new Query()); + HitGroup myChainGroup = (HitGroup) result.hits().get(0); + assertThat(myChainGroup.getId(), is(new URI("source:myChain"))); + assertThat(myChainGroup.get(0).getId(), is(new URI("myHit"))); + } + + static class TestTargetSelector implements TargetSelector { + String keyName = getClass().getName(); + + @Override + public Collection> getTargets(Query query, ChainRegistry searcherChainRegistry) { + return Arrays.asList( + new FederationTarget<>(new Chain<>("myChain", Collections.emptyList()), new FederationOptions(), "hello")); + } + + @Override + public void modifyTargetQuery(FederationTarget target, Query query) { + checkTarget(target); + query.properties().set(keyName, "called"); + } + + @Override + public void modifyTargetResult(FederationTarget target, Result result) { + checkTarget(target); + assertThat(result.getQuery().properties().getString(keyName), is("called")); + result.hits().add(new Hit("myHit")); + } + + private void checkTarget(FederationTarget target) { + assertThat(target.getCustomData(), is("hello")); + assertThat(target.getChain().getId(), is(ComponentId.fromString("myChain"))); + } + } + + static class TestMultipleTargetSelector implements TargetSelector { + String keyName = getClass().getName(); + + @Override + public Collection> getTargets(Query query, ChainRegistry searcherChainRegistry) { + return Arrays.asList(createTarget(1), createTarget(2)); + } + + private FederationTarget createTarget(int number) { + return new FederationTarget<>(new Chain<>("chain" + number, Collections.emptyList()), + new FederationOptions(), + "custom-data:" + number); + } + + @Override + public void modifyTargetQuery(FederationTarget target, Query query) { + query.properties().set(keyName, "modifyTargetQuery:" + target.getCustomData()); + } + + @Override + public void modifyTargetResult(FederationTarget target, Result result) { + Hit hit = new Hit("MyHit" + target.getCustomData()); + hit.setField("data", result.getQuery().properties().get(keyName)); + result.hits().add(hit); + } + } + + @Test + public void target_selectors_can_have_multiple_targets() { + ComponentId targetSelectorId = ComponentId.fromString("TestMultipleTargetSelector"); + ComponentRegistry targetSelectors = new ComponentRegistry<>(); + targetSelectors.register(targetSelectorId, new TestMultipleTargetSelector()); + + FederationSearcher searcher = new FederationSearcher( + new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), + new StrictContractsConfig(new StrictContractsConfig.Builder()), + targetSelectors); + + Result result = new Execution(searcher, Context.createContextStub()).search(new Query()); + + Iterator hitsIterator = result.hits().deepIterator(); + Hit hit1 = hitsIterator.next(); + Hit hit2 = hitsIterator.next(); + + assertThat(hit1.getSource(), is("chain1")); + assertThat(hit2.getSource(), is("chain2")); + + assertThat((String)hit1.getField("data"), is("modifyTargetQuery:custom-data:1")); + assertThat((String)hit2.getField("data"), is("modifyTargetQuery:custom-data:2")); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java new file mode 100644 index 00000000000..bc00890624b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java @@ -0,0 +1,411 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.StrictContractsConfig; +import com.yahoo.search.federation.selection.TargetSelector; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.test.QueryTestCase; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.*; +import static com.yahoo.search.federation.StrictContractsConfig.PropagateSourceProperties; + +/** + * Test for federation searcher. The searcher is also tested in + * com.yahoo.prelude.searcher.test.BlendingSearcherTestCase. + * + * @author Arne Bergene Fossaa + */ +@SuppressWarnings("deprecation") +public class FederationSearcherTestCase { + + static final String SOURCE1 = "source1"; + static final String SOURCE2 = "source2"; + + public static class TwoSourceChecker extends TraceVisitor { + public boolean traceFromSource1 = false; + public boolean traceFromSource2 = false; + + @Override + public void visit(TraceNode node) { + if (SOURCE1.equals(node.payload())) { + traceFromSource1 = true; + } else if (SOURCE2.equals(node.payload())) { + traceFromSource2 = true; + } + } + + } + + private FederationConfig.Builder builder; + private SearchChainRegistry chainRegistry; + + @Before + public void setUp() throws Exception { + builder = new FederationConfig.Builder(); + chainRegistry = new SearchChainRegistry(); + } + + @After + public void tearDown() throws Exception { + builder = null; + chainRegistry = null; + } + + private void addChained(final Searcher searcher, final String sourceName) { + builder.target(new FederationConfig.Target.Builder(). + id(sourceName). + searchChain(new FederationConfig.Target.SearchChain.Builder(). + searchChainId(sourceName). + timeoutMillis(10000). + useByDefault(true)) + ); + chainRegistry.register(new ComponentId(sourceName), + createSearchChain(new ComponentId(sourceName), searcher)); + } + + private Searcher createFederationSearcher() { + return buildFederation(new StrictContractsConfig(new StrictContractsConfig.Builder())); + } + + private Searcher createFederationSearcher(PropagateSourceProperties.Enum propagateSourceProperties) { + return buildFederation(new StrictContractsConfig(new StrictContractsConfig.Builder().propagateSourceProperties(propagateSourceProperties))); + } + + private Searcher createStrictFederationSearcher() { + StrictContractsConfig.Builder builder = new StrictContractsConfig.Builder(); + builder.searchchains(true); + final StrictContractsConfig contracts = new StrictContractsConfig(builder); + return buildFederation(contracts); + } + + private Searcher buildFederation(final StrictContractsConfig contracts) + throws RuntimeException { + + return new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry()); + } + + private SearchChain createSearchChain(final ComponentId chainId, + final Searcher searcher) { + return new SearchChain(chainId, searcher); + } + + @Test + public void testQueryProfileNestedReferencing() { + addChained(new MockSearcher(), "mySource1"); + addChained(new MockSearcher(), "mySource2"); + Chain mainChain = new Chain<>("default", createFederationSearcher()); + + QueryProfile defaultProfile = new QueryProfile("default"); + defaultProfile.set("source.mySource1.hits", "%{hits}", (QueryProfileRegistry)null); + defaultProfile.freeze(); + Query q = new Query(QueryTestCase.httpEncode("?query=test"), defaultProfile.compile(null)); + + Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); + assertNull(result.hits().getError()); + assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); + } + + @Test + public void testTraceTwoSources() { + final Chain mainChain = twoTracingSources(false); + + final Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1")); + + final Execution execution = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)); + final Result result = execution.search(q); + assertNull(result.hits().getError()); + TwoSourceChecker lookForTraces = new TwoSourceChecker(); + execution.trace().accept(lookForTraces); + assertTrue(lookForTraces.traceFromSource1); + assertTrue(lookForTraces.traceFromSource2); + } + + private Chain twoTracingSources(boolean strictContracts) { + addChained(new Searcher() { + @Override + public Result search(Query query, Execution execution) { + query.trace(SOURCE1, 1); + return execution.search(query); + } + + }, SOURCE1); + + addChained(new Searcher() { + @Override + public Result search(Query query, Execution execution) { + query.trace(SOURCE2, 1); + return execution.search(query); + } + + }, SOURCE2); + + final Chain mainChain = new Chain<>("default", + new FederationSearcher(new FederationConfig(builder), + new StrictContractsConfig( + new StrictContractsConfig.Builder().searchchains(strictContracts)), + new ComponentRegistry<>())); + return mainChain; + } + + @Test + public void testTraceOneSourceNoCloning() { + final Chain mainChain = twoTracingSources(true); + + final Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1&sources=source1")); + + final Execution execution = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)); + final Result result = execution.search(q); + assertNull(result.hits().getError()); + TwoSourceChecker lookForTraces = new TwoSourceChecker(); + execution.trace().accept(lookForTraces); + assertTrue(lookForTraces.traceFromSource1); + assertFalse(lookForTraces.traceFromSource2); + } + + @Test + public void testTraceOneSourceWithCloning() { + final Chain mainChain = twoTracingSources(false); + + final Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1&sources=source1")); + + final Execution execution = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)); + final Result result = execution.search(q); + assertNull(result.hits().getError()); + TwoSourceChecker lookForTraces = new TwoSourceChecker(); + execution.trace().accept(lookForTraces); + assertTrue(lookForTraces.traceFromSource1); + assertFalse(lookForTraces.traceFromSource2); + + } + + + @Test + public void testPropertyPropagation() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.ALL); + + assertEquals("source:mySource1", result.hits().get(0).getId() + .stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId() + .stringValue()); + assertEquals("nalle", result.hits().get(0).getQuery().getPresentation() + .getSummary()); + assertNull(result.hits().get(1).getQuery().getPresentation() + .getSummary()); + + } + + private Result searchWithPropertyPropagation(PropagateSourceProperties.Enum propagateSourceProperties) { + addChained(new MockSearcher(), "mySource1"); + addChained(new MockSearcher(), "mySource2"); + final Chain mainChain = new Chain<>("default", createFederationSearcher(propagateSourceProperties)); + + final Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle")); + + final Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); + assertNull(result.hits().getError()); + return result; + } + + @Test + public void testDisablePropertyPropagation() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.NONE); + + assertNull(result.hits().get(0).getQuery().getPresentation() + .getSummary()); + } + + @Test + public void testNoCloning() { + final String sourceName = "cloningcheck"; + Query query = new Query(QueryTestCase.httpEncode("?query=test&sources=" + sourceName)); + addChained(new QueryCheckSearcher(query), sourceName); + addChained(new MockSearcher(), "mySource1"); + Chain mainChain = new Chain<>("default", createStrictFederationSearcher()); + Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(query); + HitGroup h = (HitGroup) result.hits().get(0); + assertNull(h.getErrorHit()); + assertSame(QueryCheckSearcher.OK, h.get(0).getField(QueryCheckSearcher.STATUS)); + + mainChain = new Chain<>("default", createFederationSearcher()); + result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(query); + h = (HitGroup) result.hits().get(0); + assertSame(QueryCheckSearcher.FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY, + h.getError().getDetailedMessage()); + + query = new Query(QueryTestCase.httpEncode("?query=test&sources=" + sourceName + ",mySource1")); + addChained(new QueryCheckSearcher(query), sourceName); + result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(query); + h = (HitGroup) result.hits().get(0); + assertEquals("source:" + sourceName, h.getId().stringValue()); + assertSame(QueryCheckSearcher.FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY, + h.getError().getDetailedMessage()); + assertEquals("source:mySource1", result.hits().get(1).getId() + .stringValue()); + } + + @Test + public void testTopLevelHitGroupFieldPropagation() { + addChained(new MockSearcher(), "mySource1"); + addChained(new AnotherMockSearcher(), "mySource2"); + Chain mainChain = new Chain<>("default", createFederationSearcher()); + + Query q = new Query("?query=test"); + + Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); + assertNull(result.hits().getError()); + assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); + assertEquals( + AnotherMockSearcher.IS_THIS_PROPAGATED, + result.hits().get(1).getField(AnotherMockSearcher.PROPAGATION_KEY)); + } + + private static class MockSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + String sourceName = query.properties().getString("sourceName", "unknown"); + Result result = new Result(query); + for (int i = 1; i <= query.getHits(); i++) { + final Hit hit = new Hit(sourceName + ":" + i, 1d / i); + hit.setSource(sourceName); + result.hits().add(hit); + } + return result; + } + + } + + private static class SleepingMockSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + try { + Thread.sleep(100); + } catch (Exception e) { + throw new RuntimeException(e); + } + return execution.search(query); + } + } + + + private static class AnotherMockSearcher extends Searcher { + + private static final String PROPAGATION_KEY = "hello"; + private static final String IS_THIS_PROPAGATED = "is this propagated?"; + + @Override + public Result search(final Query query, final Execution execution) { + final Result result = new Result(query); + result.hits().setField(PROPAGATION_KEY, IS_THIS_PROPAGATED); + return result; + } + } + + @Test + public void testProviderSelectionFromQueryProperties() { + SearchChainRegistry registry = new SearchChainRegistry(); + registry.register(new Chain<>("provider1", new MockProvider("provider1"))); + registry.register(new Chain<>("provider2", new MockProvider("provider2"))); + registry.register(new Chain<>("default", createMultiProviderFederationSearcher())); + assertSelects("provider1", registry); + assertSelects("provider2", registry); + } + + private void assertSelects(String providerName, SearchChainRegistry registry) { + QueryProfile profile = new QueryProfile("test"); + profile.set("source.news.provider", providerName, (QueryProfileRegistry)null); + Query query = new Query(QueryTestCase.httpEncode("?query=test&model.sources=news"), profile.compile(null)); + Result result = new Execution(registry.getComponent("default"), Execution.Context.createContextStub(registry, null)).search(query); + assertEquals(1, result.hits().size()); + assertNotNull(result.hits().get(providerName + ":1")); + } + + private FederationSearcher createMultiProviderFederationSearcher() { + final FederationOptions options = new FederationOptions(); + final SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + + final ComponentId provider1 = new ComponentId("provider1"); + final ComponentId provider2 = new ComponentId("provider2"); + final ComponentId news = new ComponentId("news"); + builder.addSearchChain(provider1, options, + Collections. emptyList()); + builder.addSearchChain(provider2, options, + Collections. emptyList()); + builder.addSourceForProvider(news, provider1, provider1, true, options, + Collections. emptyList()); + builder.addSourceForProvider(news, provider2, provider2, false, + options, Collections. emptyList()); + + return new FederationSearcher(new ComponentId("federation"), builder.build()); + } + + private static class MockProvider extends Searcher { + + private final String name; + + public MockProvider(final String name) { + this.name = name; + } + + @Override + public Result search(final Query query, final Execution execution) { + final Result result = new Result(query); + result.hits().add(new Hit(name + ":1")); + return result; + } + + } + + private static class QueryCheckSearcher extends Searcher { + private static final String STATUS = "status"; + public static final String FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY = "FederationSearcher has cloned the query."; + public static final String OK = "Got the correct query."; + private final Query query; + + QueryCheckSearcher(final Query query) { + this.query = query; + } + + @Override + public Result search(final Query query, final Execution execution) { + final Result result = new Result(query); + if (query != this.query) { + result.hits().addError(ErrorMessage + .createErrorInPluginSearcher(FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY)); + } else { + final Hit h = new Hit("QueryCheckSearcher status hit"); + h.setField(STATUS, OK); + result.hits().add(h); + } + return result; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java new file mode 100644 index 00000000000..7b0451a01ba --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; + +import java.util.Collections; + +/** +* @author tonytv +*/ +class FederationTester { + SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + SearchChainRegistry registry = new SearchChainRegistry(); + + Execution execution; + + void addSearchChain(String id, Searcher... searchers) { + addSearchChain(id, federationOptions(), searchers); + } + + void addSearchChain(String id, FederationOptions federationOptions, Searcher... searchers) { + ComponentId searchChainId = ComponentId.fromString(id); + + builder.addSearchChain(searchChainId, federationOptions, Collections.emptyList()); + + Chain chain = new Chain<>(searchChainId, searchers); + registry.register(chain); + } + + public void addOptionalSearchChain(String id, Searcher... searchers) { + addSearchChain(id, federationOptions().setOptional(true), searchers); + } + + private FederationOptions federationOptions() { + int preventTimeout = 24 * 60 * 60 * 1000; + return new FederationOptions().setUseByDefault(true).setTimeoutInMilliseconds(preventTimeout); + } + + FederationSearcher buildFederationSearcher() { + return new FederationSearcher(ComponentId.fromString("federation"), builder.build()); + } + + public Result search() { + return search(new Query()); + } + + public Result search(Query query) { + execution = createExecution(); + return execution.search(query); + } + + public Result searchAndFill() { + Result result = search(); + fill(result); + return result; + } + + private Execution createExecution() { + registry.freeze(); + return new Execution(new Chain(buildFederationSearcher()), Execution.Context.createContextStub(registry, null)); + } + + public void fill(Result result) { + execution.fill(result, "default"); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java new file mode 100644 index 00000000000..dcbbb217c7d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.StringStartsWith.startsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class HitCountTestCase { + + @Test + public void require_that_offset_and_hits_are_adjusted_when_federating() { + final int chain1RelevanceMultiplier = 1; + final int chain2RelevanceMultiplier = 10; + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitsWithRelevanceSearcher("chain1", chain1RelevanceMultiplier)); + tester.addSearchChain("chain2", new AddHitsWithRelevanceSearcher("chain2", chain2RelevanceMultiplier)); + + Query query = new Query(); + query.setHits(5); + + query.setOffset(0); + assertAllHitsFrom("chain2", flattenAndTrim(tester.search(query))); + + query.setOffset(5); + assertAllHitsFrom("chain1", flattenAndTrim(tester.search(query))); + } + + @Test + public void require_that_hit_counts_are_merged() { + final long chain1TotalHitCount = 3; + final long chain1DeepHitCount = 5; + + final long chain2TotalHitCount = 7; + final long chain2DeepHitCount = 11; + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new SetHitCountsSearcher(chain1TotalHitCount, chain1DeepHitCount)); + tester.addSearchChain("chain2", new SetHitCountsSearcher(chain2TotalHitCount, chain2DeepHitCount)); + + Result result = tester.searchAndFill(); + + assertThat(result.getTotalHitCount(), is(chain1TotalHitCount + chain2TotalHitCount)); + assertThat(result.getDeepHitCount(), is(chain1DeepHitCount + chain2DeepHitCount)); + } + + @Test + public void require_that_logging_hit_is_populated_with_result_count() { + final long chain1TotalHitCount = 9; + final long chain1DeepHitCount = 14; + + final long chain2TotalHitCount = 11; + final long chain2DeepHitCount = 15; + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", + new SetHitCountsSearcher(chain1TotalHitCount, chain1DeepHitCount)); + + tester.addSearchChain("chain2", + new SetHitCountsSearcher(chain2TotalHitCount, chain2DeepHitCount), + new AddHitsWithRelevanceSearcher("chain1", 2)); + + Query query = new Query(); + query.setOffset(2); + query.setHits(7); + Result result = tester.search(); + List metaHits = getFirstMetaHitInEachGroup(result); + + Hit first = metaHits.get(0); + assertEquals(chain1TotalHitCount, first.getField("count_total")); + assertEquals(chain1TotalHitCount, first.getField("count_total")); + assertEquals(1, first.getField("count_first")); + assertEquals(0, first.getField("count_last")); + + Hit second = metaHits.get(1); + assertEquals(chain2TotalHitCount, second.getField("count_total")); + assertEquals(chain2TotalHitCount, second.getField("count_total")); + assertEquals(1, second.getField("count_first")); + assertEquals(AddHitsWithRelevanceSearcher.numHitsAdded, second.getField("count_last")); + + } + + private List getFirstMetaHitInEachGroup(Result result) { + List metaHits = new ArrayList<>(); + for (Hit topLevelHit : result.hits()) { + if (topLevelHit instanceof HitGroup) { + for (Hit hit : (HitGroup)topLevelHit) { + if (hit.isMeta()) { + metaHits.add(hit); + break; + } + } + } + } + return metaHits; + } + + private void assertAllHitsFrom(String chainName, HitGroup flattenedHits) { + for (Hit hit : flattenedHits) { + assertThat(hit.getId().toString(), startsWith(chainName)); + } + } + + private HitGroup flattenAndTrim(Result result) { + HitGroup flattenedHits = new HitGroup(); + result.setQuery(result.getQuery()); + flatten(result.hits(), flattenedHits); + + flattenedHits.trim(result.getQuery().getOffset(), result.getQuery().getHits()); + return flattenedHits; + } + + private void flatten(HitGroup hits, HitGroup flattenedHits) { + for (Hit hit : hits) { + if (hit instanceof HitGroup) { + flatten((HitGroup) hit, flattenedHits); + } else { + flattenedHits.add(hit); + } + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java b/container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java new file mode 100644 index 00000000000..81a3007735c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author tonytv + */ +class SetHitCountsSearcher extends Searcher { + + private final long totalHitCount; + private final long deepHitCount; + + public SetHitCountsSearcher(long totalHitCount, long deepHitCount) { + this.totalHitCount = totalHitCount; + this.deepHitCount = deepHitCount; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(createLoggingHit()); + + result.setTotalHitCount(totalHitCount); + result.setDeepHitCount(deepHitCount); + return result; + } + + private Hit createLoggingHit() { + Hit hit = new Hit("SetHitCountSearcher"); + hit.setMeta(true); + hit.types().add("logging"); + return hit; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java new file mode 100644 index 00000000000..2868d69457b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.*; +import com.yahoo.search.Query; +import com.yahoo.search.federation.vespa.QueryMarshaller; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class QueryMarshallerTestCase { + + private static final Linguistics linguistics = new SimpleLinguistics(); + + @Test + public void testCommonCommonCase() { + AndItem root = new AndItem(); + addThreeWords(root); + assertEquals("a AND b AND c", new QueryMarshaller().marshal(root)); + } + + @Test + public void testPhrase() { + PhraseItem root = new PhraseItem(); + root.setIndexName("habla"); + addThreeWords(root); + assertEquals("habla:\"a b c\"", new QueryMarshaller().marshal(root)); + } + + @Test + public void testPhraseDefaultIndex() { + PhraseItem root = new PhraseItem(); + addThreeWords(root); + assertEquals("\"a b c\"", new QueryMarshaller().marshal(root)); + } + + @Test + public void testLittleMoreComplex() { + AndItem root = new AndItem(); + addThreeWords(root); + OrItem ambig = new OrItem(); + root.addItem(ambig); + addThreeWords(ambig); + AndItem but = new AndItem(); + addThreeWords(but); + ambig.addItem(but); + assertEquals("a AND b AND c AND ( a OR b OR c OR ( a AND b AND c ) )", + new QueryMarshaller().marshal(root)); + } + + @Test + public void testRank() { + RankItem root = new RankItem(); + addThreeWords(root); + assertEquals("a RANK b RANK c", new QueryMarshaller().marshal(root)); + } + + @Test + public void testNear() { + NearItem near = new NearItem(3); + addThreeWords(near); + assertEquals("a NEAR(3) b NEAR(3) c", new QueryMarshaller().marshal(near)); + } + + @Test + public void testONear() { + ONearItem oNear = new ONearItem(3); + addThreeWords(oNear); + assertEquals("a ONEAR(3) b ONEAR(3) c", new QueryMarshaller().marshal(oNear)); + } + + private void addThreeWords(CompositeItem root) { + root.addItem(new WordItem("a")); + root.addItem(new WordItem("b")); + root.addItem(new WordItem("c")); + } + + @Test + public void testNegativeGroupedTerms() { + testQueryString(new QueryMarshaller(), "a -(b c) -(d e)", + "a ANDNOT ( b AND c ) ANDNOT ( d AND e )"); + } + + @Test + public void testPositiveGroupedTerms() { + testQueryString(new QueryMarshaller(), "a (b c)", "a AND ( b OR c )"); + } + + @Test + public void testInt() { + testQueryString(new QueryMarshaller(), "yahoo 123", "yahoo AND 123"); + } + + @Test + public void testCJKOneWord() { + testQueryString(new QueryMarshaller(), "天龍人"); + } + + @Test + public void testTwoWords() { + testQueryString(new QueryMarshaller(), "John Smith", "John AND Smith", null, new SimpleLinguistics()); + } + + @Test + public void testTwoWordsInPhrase() { + testQueryString(new QueryMarshaller(), "\"John Smith\"", "\"John Smith\"", null, new SimpleLinguistics()); + } + + @Test + public void testCJKTwoSentences() { + testQueryString(new QueryMarshaller(), "是不是這樣的夜晚 你才會這樣地想起我", "是不是這樣的夜晚 AND 你才會這樣地想起我"); + } + + @Test + public void testCJKTwoSentencesWithLanguage() { + testQueryString(new QueryMarshaller(), "助妳好孕 生1胎北市發2萬", "助妳好孕 AND 生1胎北市發2萬", "zh-Hant"); + } + + @Test + public void testCJKTwoSentencesInPhrase() { + QueryMarshaller marshaller = new QueryMarshaller(); + testQueryString(marshaller, "\"助妳好孕 生1胎北市發2萬\"", "\"助妳好孕 生1胎北市發2萬\"", "zh-Hant"); + testQueryString(marshaller, "\"是不是這樣的夜晚 你才會這樣地想起我\"", "\"是不是這樣的夜晚 你才會這樣地想起我\""); + } + + @Test + public void testCJKMultipleSentences() { + testQueryString(new QueryMarshaller(), "염부장님과 함께했던 좋은 추억들은", "염부장님과 AND 함께했던 AND 좋은 AND 추억들은"); + } + + @Test + public void testIndexRestriction() { + /** ticket 3707606, comment #29 */ + testQueryString(new QueryMarshaller(), "site:nytimes.com", "site:\"nytimes com\""); + } + + private void testQueryString(QueryMarshaller marshaller, String uq) { + testQueryString(marshaller, uq, uq, null); + } + + private void testQueryString(QueryMarshaller marshaller, String uq, String mq) { + testQueryString(marshaller, uq, mq, null); + } + + private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang) { + testQueryString(marshaller, uq, mq, lang, linguistics); + } + + private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang, Linguistics linguistics) { + Query query = new Query("/?query=" + QueryTestCase.httpEncode(uq) + ((lang != null) ? "&language=" + lang : "")); + query.getModel().setExecution(new Execution(new Execution.Context(null, new IndexFacts(), null, null, linguistics))); + assertEquals(mq, marshaller.marshal(query.getModel().getQueryTree().getRoot())); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java new file mode 100644 index 00000000000..9135984b26b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.search.Query; +import com.yahoo.search.federation.vespa.VespaSearcher; +import java.util.Map; + +/** + * Tests that source and backend specific parameters from the query are added correctly to the backend requests + * + * @author Jon Bratseth + */ +public class QueryParametersTestCase extends junit.framework.TestCase { + + public void testQueryParameters() { + Query query=new Query(); + query.properties().set("a","a-value"); + query.properties().set("b.c","b.c-value"); + query.properties().set("source.otherSource.d","d-value"); + query.properties().set("source.testSource.e","e-value"); + query.properties().set("source.testSource.f.g","f.g-value"); + query.properties().set("provider.testProvider.h","h-value"); + query.properties().set("provider.testProvider.i.j","i.j-value"); + + query.properties().set("sourceName","testSource"); // Done by federation searcher + query.properties().set("providerName","testProvider"); // Done by federation searcher + + VespaSearcher searcher=new VespaSearcher("testProvider","",0,""); + Map parameters=searcher.getQueryMap(query); + searcher.deconstruct(); + + assertEquals(9, parameters.size()); // 5 standard + the appropriate 4 of the above + assertEquals(parameters.get("e"),"e-value"); + assertEquals(parameters.get("f.g"),"f.g-value"); + assertEquals(parameters.get("h"),"h-value"); + assertEquals(parameters.get("i.j"),"i.j-value"); + } + +} + diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java new file mode 100644 index 00000000000..8cec6c64554 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import java.util.Iterator; + +import junit.framework.TestCase; + +import com.yahoo.net.URI; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.federation.vespa.ResultBuilder; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.HitGroup; + +/** + * Test XML parsing of results. + * + * @author Steinar Knutsen + */ +@SuppressWarnings("deprecation") +public class ResultBuilderTestCase extends TestCase { + + public ResultBuilderTestCase (String name) { + super(name); + } + + private boolean quickCompare(double a, double b) { + double z = Math.min(Math.abs(a), Math.abs(b)); + if (Math.abs((a - b)) < (z / 1e14)) { + return true; + } else { + return false; + } + } + + public void testSimpleResult() { + boolean gotErrorDetails = false; + ResultBuilder r = new ResultBuilder(); + Result res = r.parse("file:src/test/java/com/yahoo/prelude/searcher/test/testhit.xml", new Query("?query=a")); + assertEquals(3, res.getConcreteHitCount()); + assertEquals(4, res.getHitCount()); + ErrorHit e = (ErrorHit) res.hits().get(0); + // known problem, if the same error is the main error is + // in details, it'll be added twice. Not sure how to fix that, + // because old Vespa systems give no error details, and there + // is no way of nuking an existing error if the details exist. + for (Iterator i = e.errorIterator(); i.hasNext();) { + ErrorMessage err = (ErrorMessage) i.next(); + assertEquals(5, err.getCode()); + String details = err.getDetailedMessage(); + if (details != null) { + gotErrorDetails = true; + assertEquals("An error as ordered", details.trim()); + } + } + assertTrue("Error details are missing", gotErrorDetails); + assertEquals(new URI("http://def"), res.hits().get(1).getId()); + assertEquals("test/stuff\\tsome/other", res.hits().get(2).getField("category")); + assertEquals("habla" + + "blbl
<>&fdlkkgj</field>;lk" + + "", res.hits().get(3).getField("annoying").toString()); + } + + public void testNestedResult() { + ResultBuilder r = new ResultBuilder(); + Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml", new Query("?query=a")); + assertNull(res.hits().getError()); + assertEquals(3, res.hits().size()); + assertEquals("ABCDEFGHIJKLMNOPQRSTUVWXYZ", res.hits().get(0).getField("guid").toString()); + HitGroup g1 = (HitGroup) res.hits().get(1); + HitGroup g2 = (HitGroup) res.hits().get(2); + assertEquals(15, g1.size()); + assertEquals("reward_for_thumb", g1.get(1).getField("id").toString()); + assertEquals(10, g2.size()); + HitGroup g3 = (HitGroup) g2.get(3); + assertEquals("badge", g3.getTypeString()); + assertEquals(2, g3.size()); + assertEquals("badge/Topic Explorer 5", g3.get(0).getField("name").toString()); + } + + public void testWeirdDocumentID() { + ResultBuilder r = new ResultBuilder(); + Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml", new Query("?query=a")); + assertNull(res.hits().getError()); + assertEquals(3, res.hits().size()); + assertEquals(new URI("nalle"), res.hits().get(0).getId()); + assertEquals(new URI("tralle"), res.hits().get(1).getId()); + assertEquals(new URI("kalle"), res.hits().get(2).getId()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java new file mode 100644 index 00000000000..a1c3529e2e4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.vespa.VespaSearcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class VespaIntegrationTestCase extends junit.framework.TestCase { + + // TODO: Setup the answering vespa searcher from this test.... + public void testIt() { + if (System.currentTimeMillis() > 0) return; + Chain chain=new Chain<>(new VespaSearcher("test","example.yahoo.com",19010,"")); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=test")); + assertEquals(23,result.hits().size()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java new file mode 100644 index 00000000000..63da6adca77 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java @@ -0,0 +1,229 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.prelude.query.*; +import com.yahoo.search.Query; +import com.yahoo.search.federation.vespa.VespaSearcher; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import junit.framework.TestCase; +import org.apache.http.HttpEntity; +import java.io.IOException; +import java.net.URI; + +/** + * Check query marshaling in VespaSearcher works... and stuff... + * + * @author Steinar Knutsen + */ +public class VespaSearcherTestCase extends TestCase { + + // TODO: More tests + + private VespaSearcher searcher; + + protected @Override void setUp() { + searcher = new VespaSearcher("cache1","",0,""); + } + + protected @Override void tearDown() { + searcher.deconstruct(); + } + + public void testMarshalQuery() { + RankItem root = new RankItem(); + QueryTree r = new QueryTree(root); + AndItem recall = new AndItem(); + PhraseItem usual = new PhraseItem(); + PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); + WordItem filterWord = new WordItem("silly"); + + filterPhrase.setFilter(true); + filterWord.setFilter(true); + + root.addItem(recall); + usual.addItem(new WordItem("new")); + usual.addItem(new WordItem("york")); + recall.addItem(usual); + recall.addItem(new WordItem("shoes")); + root.addItem(new WordItem("nike")); + root.addItem(new WordItem("adidas")); + root.addItem(filterPhrase); + recall.addItem(filterWord); + + assertEquals("( \"new york\" AND shoes AND silly ) RANK nike RANK adidas RANK \"bloody expensive\"", searcher.marshalQuery(r)); + } + + public void testMarshalQuerySmallTree() { + RankItem root = new RankItem(); + QueryTree r = new QueryTree(root); + AndItem recall = new AndItem(); + PhraseItem usual = new PhraseItem(); + PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); + WordItem filterWord = new WordItem("silly"); + + filterPhrase.setFilter(true); + filterWord.setFilter(true); + + root.addItem(recall); + usual.addItem(new WordItem("new")); + usual.addItem(new WordItem("york")); + recall.addItem(usual); + recall.addItem(new WordItem("shoes")); + root.addItem(filterPhrase); + recall.addItem(filterWord); + + assertEquals("( \"new york\" AND shoes AND silly ) RANK \"bloody expensive\"", searcher.marshalQuery(r)); + // TODO: Switch to this 2-way check rather than just 1-way and then also make this actually treat filter terms correctly + // assertMarshals(root) + } + + public void testWandMarshalling() { + WeakAndItem root = new WeakAndItem(); + root.setN(32); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("b")); + root.addItem(new WordItem("c")); + assertMarshals(root); + } + + public void testWandMarshalling2() { + // AND (WAND(10) a!1 the!10) source:yahoonews + AndItem root = new AndItem(); + WeakAndItem wand = new WeakAndItem(10); + wand.addItem(newWeightedWordItem("a",1)); + wand.addItem(newWeightedWordItem("the",10)); + root.addItem(wand); + root.addItem(new WordItem("yahoonews","source")); + assertMarshals(root); + } + + private WordItem newWeightedWordItem(String word,int weight) { + WordItem wordItem=new WordItem(word); + wordItem.setWeight(weight); + return wordItem; + } + + private void assertMarshals(Item root) { + QueryTree r = new QueryTree(root); + String marshalledQuery=searcher.marshalQuery(r); + assertEquals("Marshalled form '" + marshalledQuery + "' recreates the original", + r,parseQuery(marshalledQuery,"")); + } + + private static Item parseQuery(String query, String filter) { + Parser parser = ParserFactory.newInstance(Query.Type.ADVANCED, new ParserEnvironment()); + return parser.parse(new Parsable().setQuery(query).setFilter(filter)); + } + + public void testSourceProviderProperties() throws Exception { + /* TODO: update test + Server httpServer = new Server(); + try { + SocketConnector listener = new SocketConnector(); + listener.setHost("0.0.0.0"); + httpServer.addConnector(listener); + httpServer.setHandler(new DummyHandler()); + httpServer.start(); + + int port=httpServer.getConnectors()[0].getLocalPort(); + + List sourcesConfig = new ArrayList(); + SourcesConfig.Source sourceConfig = new SourcesConfig.Source(); + sourceConfig.chain.setValue("news"); + sourceConfig.provider.setValue("news"); + sourceConfig.id.setValue("news"); + sourceConfig.timelimit.value = 10000; + sourcesConfig.add(sourceConfig); + FederationSearcher federator = + new FederationSearcher(ComponentId.createAnonymousComponentId(), + new ArrayList(sourcesConfig)); + SearchChain mainChain=new OrderedSearchChain(federator); + + SearchChainRegistry registry=new SearchChainRegistry(); + SearchChain sourceChain=new SearchChain(new ComponentId("news"),new VespaSearcher("test","localhost",port,"")); + registry.register(sourceChain); + Query query=new Query("?query=hans&hits=20&provider.news.a=a1&source.news.b=b1"); + Result result=new Execution(mainChain,registry).search(query); + assertNull(result.hits().getError()); + Hit testHit=result.hits().get("testHit"); + assertNotNull(testHit); + assertEquals("testValue",testHit.fields().get("testField")); + assertEquals("a1",testHit.fields().get("a")); + assertEquals("b1",testHit.fields().get("b")); + } + finally { + httpServer.stop(); + } + */ + } + + public void testVespaSearcher() { + VespaSearcher v=new VespaSearcherValidatingSubclass(); + new Execution(v, Execution.Context.createContextStub()).search(new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&filter=myfilter"))); + } + + private class VespaSearcherValidatingSubclass extends VespaSearcher { + + public VespaSearcherValidatingSubclass() { + super("configId","host",80,"path"); + } + + @Override + protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { + assertEquals("http://host:80/path?query=test+RANK+myfilter&type=adv&offset=0&hits=10&presentation.format=xml",uri.toString()); + return super.getEntity(uri,requestMeta,query); + } + + } + + // used by the old testSourceProviderProperties() +// private class DummyHandler extends AbstractHandler { +// public void handle(String s, Request request, HttpServletRequest httpServletRequest, +// HttpServletResponse httpServletResponse) throws IOException, ServletException { +// +// try { +// Response httpResponse = httpServletResponse instanceof Response ? (Response) httpServletResponse : HttpConnection.getCurrentConnection().getResponse(); +// +// httpResponse.setStatus(HttpStatus.OK_200); +// httpResponse.setContentType("text/xml"); +// httpResponse.setCharacterEncoding("UTF-8"); +// Result r=new Result(new Query()); +// Hit testHit=new Hit("testHit"); +// testHit.setField("uri","testHit"); // That this is necessary is quite unfortunate... +// testHit.setField("testField","testValue"); +// // Write back all incoming properties: +// for (Object e : httpServletRequest.getParameterMap().entrySet()) { +// Map.Entry entry=(Map.Entry)e; +// testHit.setField(entry.getKey().toString(),getFirstValue(entry.getValue())); +// } +// +// r.hits().add(testHit); +// +// //StringWriter sw=new StringWriter(); +// //r.render(sw); +// //System.out.println(sw.toString()); +// +// SearchRendererAdaptor.callRender(httpResponse.getWriter(), r); +// httpResponse.complete(); +// } +// catch (Exception e) { +// System.out.println("WARNING: Could not respond to request: " + Exceptions.toMessageString(e)); +// e.printStackTrace(); +// } +// } +// +// private String getFirstValue(Object entry) { +// if (entry instanceof String[]) +// return ((String[])entry)[0].toString(); +// else +// return entry.toString(); +// } +// } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml new file mode 100644 index 00000000000..b4b5c072eca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml @@ -0,0 +1,23 @@ + + + + + nalle + 75 + 0 + + + tralle + 73 + 0 + test/stuff\tsome/other + dklf øæå sdf > & < +Ipsum, etc. + + + kalle + 75 + 0 + hablablbl
&fdlkkgj]]>;lk + + diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml new file mode 100644 index 00000000000..c935f16528f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml @@ -0,0 +1,318 @@ + + + + +ABCDEFGHIJKLMNOPQRSTUVWXYZ +zero +0 +1287600988 +1287600988 + + + +thumb +1287600992 +1287600992 +0 +zero +1 + + +reward_for_thumb +1287600992 +1287600992 +0 +zero +1 + + +undo_thumb +1287600992 +1287600992 +0 +zero +1 + + + +buzz +1287600989 +1287600989 +0 +zero +1 + + + +undo_reward_for_thumb +1287600992 +1287600992 +0 +zero +1 + + + +vote +1287600989 +1287600989 +0 +zero +1 + + + +report_abuse +1287600992 +1287600992 +0 +zero +1 + + + +reward_for_vote +1287600989 +1287600989 +0 +zero +1 + + + +signup +1287600993 +1287600993 +0 +zero +1 + + + +registered +1287600989 +1287600989 +0 +zero +1 + + + +get_points +1287600989 +1287600989 +0 +zero +1 + + + +contrib_SignedUp +1287600993 +1287600993 +0 +zero +1 + + + +contrib_AgreedToTos +1287600993 +1287600993 +500 +zero +1 + + + +Create Feature + + +0 + +1 + + + +add_theme + + +0 + +1 + + + + + + + + +badge +badge/First Feature +You’ve created your First Feature! +active +http://example.yahoo.com/1stfeature.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/25th Feature +You’ve created your 25th Feature! +active +http://example.yahoo.com/25thfeature.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/50th Feature +You’ve created your 50th Feature! +active +http://example.yahoo.com/10thfeature.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/Topic Explorer 5 +You’ve added a Feature to your 5th Topic Page! +active +http://example.yahoo.com/5thtopic.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/Topic Explorer 15 +You’ve added a Feature to your 15th Topic Page! +active +http://example.yahoo.com/15thtopic.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/Topic Explorer 30 +You’ve added a Feature to your 30th Topic Page! +active +http://example.yahoo.com/30thtopic.png +57 +57 + + + +1283981088 +topic/Jennifer_Aniston + + + + + + +badge +badge/Pollster +You’ve created your 5th Poll Feature. +active +http://example.yahoo.com/pollster.png +57 +57 + + +1283981088 +topic/Jennifer_Aniston + + + + +badge +badge/Reporter +You’ve created your 5th Article Feature. +active +http://example.yahoo.com/newsreporter.png +57 +57 + + +1283981088 +topic/Jennifer_Aniston + + + + +badge +badge/Paparazzi +You’ve created your 5th Image Feature. +active +http://example.yahoo.com/paparazzi.png +57 +57 + + +1283981088 +topic/Jennifer_Aniston + + + + +badge +badge/Video Reporter +You’ve created your 5th Video Feature. +active +http://example.yahoo.com/director.png +57 +57 + + +1283981088 +topic/Jennifer_Aniston + + + + diff --git a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore new file mode 100644 index 00000000000..e69de29bb2d -- cgit v1.2.3