summaryrefslogtreecommitdiffstats
path: root/container-search/src/test/java/com/yahoo/prelude/fastsearch
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2019-09-18 07:03:13 +0200
committerGitHub <noreply@github.com>2019-09-18 07:03:13 +0200
commit89a1b3ae32347c00e1177576668385ebfe3a6a68 (patch)
tree2856305bc733d52b7e6a26e0a36afacef8209ccb /container-search/src/test/java/com/yahoo/prelude/fastsearch
parentf6b49bf9f9e98b5778d9c8856e128a1fcdb5e017 (diff)
Revert "Revert "Balder/no more fs4 dispatching from fastsearcher""
Diffstat (limited to 'container-search/src/test/java/com/yahoo/prelude/fastsearch')
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/FS4SearchInvokerTestCase.java72
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java137
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java364
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java127
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java19
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/DispatchThread.java101
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockBackend.java53
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFDispatch.java212
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFS4ResourcePool.java63
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java176
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/NonWorkingMockFSChannel.java21
11 files changed, 13 insertions, 1332 deletions
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/FS4SearchInvokerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/FS4SearchInvokerTestCase.java
deleted file mode 100644
index b9119528490..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/FS4SearchInvokerTestCase.java
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-package com.yahoo.prelude.fastsearch;
-
-import com.yahoo.fs4.BasicPacket;
-import com.yahoo.fs4.mplex.FS4Channel;
-import com.yahoo.fs4.mplex.InvalidChannelException;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.dispatch.InterleavedSearchInvoker;
-import com.yahoo.search.dispatch.MockSearchCluster;
-import com.yahoo.search.dispatch.ResponseMonitor;
-import com.yahoo.search.searchchain.Execution;
-import org.hamcrest.Matchers;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Optional;
-
-import static org.junit.Assert.assertThat;
-
-public class FS4SearchInvokerTestCase {
- @SuppressWarnings("resource")
- @Test
- public void testThatConnectionErrorsAreReportedImmediately() throws IOException {
- var query = new Query("?");
- query.setTimeout(1000);
-
- var searcher = mockSearcher();
- var cluster = new MockSearchCluster("?", 1, 1);
- var fs4invoker = new FS4SearchInvoker(searcher, query, mockFailingChannel(), Optional.empty());
- var interleave = new InterleavedSearchInvoker(Collections.singleton(fs4invoker), cluster, null);
-
- long start = System.currentTimeMillis();
- interleave.search(query, null);
- long elapsed = System.currentTimeMillis() - start;
-
- assertThat("Connection error should fail fast", elapsed, Matchers.lessThan(500L));
- }
-
- private static VespaBackEndSearcher mockSearcher() {
- return new VespaBackEndSearcher() {
- @Override
- protected Result doSearch2(Query query, Execution execution) {
- return null;
- }
-
- @Override
- protected void doPartialFill(Result result, String summaryClass) {}
- };
- }
-
- private static FS4Channel mockFailingChannel() {
- return new FS4Channel() {
- @Override
- public boolean sendPacket(BasicPacket packet) throws InvalidChannelException, IOException {
- // pretend there's a connection error
- return false;
- }
-
- @Override
- public void setQuery(Query q) {}
-
- @Override
- public void setResponseMonitor(ResponseMonitor<FS4Channel> monitor) {}
-
- @Override
- public void close() {}
- };
- }
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java
deleted file mode 100644
index b0662a93f62..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java
+++ /dev/null
@@ -1,137 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.fastsearch.test;
-
-import com.yahoo.prelude.fastsearch.FastHit;
-import com.yahoo.search.Result;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests that FastSearcher will bypass dispatch when the conditions are right
- *
- * @author bratseth
- */
-public class DirectSearchTestCase {
-
- @Test
- public void testDirectSearchEnabled() {
- FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0");
- tester.search("?query=test&dispatch.direct=true");
- assertEquals("The FastSearcher has used the local search node connection", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- }
-
- @Test
- public void testDirectSearchDisabled() {
- FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0");
- tester.search("?query=test&dispatch.direct=false&dispatch.internal=false");
- assertEquals(0, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- }
-
- @Test
- public void testDirectSearchEnabledByDefault() {
- FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0");
- tester.search("?query=test");
- assertEquals("The FastSearcher has used the local search node connection", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- }
-
- @Test
- public void testNoDirectSearchWhenMoreSearchNodesThanContainers() {
- FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0", "otherhost:9999:1");
- tester.search("?query=test&dispatch.direct=true&dispatch.internal=false");
- assertEquals(0, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- }
-
- @Test
- public void testDirectSearchWhenMultipleGroupsAndEnoughContainers() {
- FastSearcherTester tester = new FastSearcherTester(2, FastSearcherTester.selfHostname + ":9999:0", "otherhost:9999:1");
- tester.search("?query=test&dispatch.direct=true");
- assertEquals(1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- }
-
- @Test
- public void testDirectSearchSummaryFetchGoToLocalNode() {
- FastSearcherTester tester = new FastSearcherTester(2, "otherhost:9999:1", FastSearcherTester.selfHostname + ":9999:0");
- int localDistributionKey = tester.dispatcher().searchCluster().nodesByHost().get(FastSearcherTester.selfHostname).asList().get(0).key();
- assertEquals(1, localDistributionKey);
- Result result = tester.search("?query=test&dispatch.direct=true");
- assertEquals(1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- FastHit hit = (FastHit)result.hits().get(0);
- assertEquals(localDistributionKey, hit.getDistributionKey());
- }
-
- @Test
- public void testNoDirectSearchWhenMultipleNodesPerGroup() {
- FastSearcherTester tester = new FastSearcherTester(2, FastSearcherTester.selfHostname + ":9999:0", "otherhost:9999:0");
- tester.search("?query=test&dispatch.direct=true&dispatch.internal=false");
- assertEquals(0, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- }
-
- @Test
- public void testNoDirectSearchWhenLocalNodeIsDown() {
- FastSearcherTester tester = new FastSearcherTester(2, FastSearcherTester.selfHostname + ":9999:0", "otherhost:9999:1");
- assertTrue(tester.vipStatus().isInRotation());
- tester.setResponding(FastSearcherTester.selfHostname, false);
- assertFalse(tester.vipStatus().isInRotation());
- assertEquals("1 ping request, 0 search requests", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.search("?query=test&dispatch.direct=true&nocache");
- assertEquals("1 ping request, 0 search requests", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.setResponding(FastSearcherTester.selfHostname, true);
- assertTrue(tester.vipStatus().isInRotation());
- assertEquals("2 ping requests, 0 search request", 2, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.search("?query=test&dispatch.direct=true&nocache");
- assertEquals("2 ping requests, 1 search request", 3, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- }
-
- @Test
- public void testNoDirectDispatchWhenInsufficientCoverage() {
- FastSearcherTester tester = new FastSearcherTester(3,
- FastSearcherTester.selfHostname + ":9999:0",
- "host1:9999:1",
- "host2:9999:2");
- double k = 38.78955; // multiply all document counts by some number > 1 to test that we compute % correctly
-
- tester.setActiveDocuments(FastSearcherTester.selfHostname, (long) (96 * k));
- tester.setActiveDocuments("host1", (long) (100 * k));
- tester.setActiveDocuments("host2", (long) (100 * k));
- assertEquals("1 ping request, 0 search requests", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.search("?query=test&dispatch.direct=true&nocache");
- assertEquals("Still 1 ping request, 0 search requests because the default coverage is 97%, and we only have 96% locally",
- 1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.waitForInRotationIs(false);
-
- tester.setActiveDocuments(FastSearcherTester.selfHostname, (long) (99 * k));
- assertEquals("2 ping request, 0 search requests", 2, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.search("?query=test&dispatch.direct=true&nocache");
- assertEquals("2 ping request, 1 search requests because we now have 99% locally",
- 3, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.waitForInRotationIs(true);
-
- tester.setActiveDocuments("host1", (long) (104 * k));
- assertEquals("2 ping request, 1 search requests", 3, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.search("?query=test&dispatch.direct=true&nocache");
- assertEquals("2 ping request, 2 search requests because 99/((104+100)/2) > 0.97",
- 4, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.waitForInRotationIs(true);
-
- tester.setActiveDocuments("host2", (long) (102 * k));
- assertEquals("2 ping request, 2 search requests", 4, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.search("?query=test&dispatch.direct=true&nocache");
- assertEquals("Still 2 ping request, 2 search requests because 99/((104+102)/2) < 0.97",
- 4, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.waitForInRotationIs(false);
- }
-
- @Test
- public void testCoverageWithSingleGroup() {
- FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0");
-
- tester.setActiveDocuments(FastSearcherTester.selfHostname, 100);
- assertEquals("1 ping request, 0 search requests", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- tester.search("?query=test&dispatch.direct=true&nocache");
- assertEquals("1 ping request, 1 search requests", 2, tester.requestCount(FastSearcherTester.selfHostname, 9999));
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
index c6e87170f07..eb4d65693bb 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java
@@ -3,64 +3,34 @@ package com.yahoo.prelude.fastsearch.test;
import com.google.common.collect.ImmutableList;
import com.yahoo.component.chain.Chain;
-import com.yahoo.config.subscription.ConfigGetter;
-import com.yahoo.container.QrConfig;
-import com.yahoo.container.handler.VipStatus;
import com.yahoo.container.protect.Error;
-import com.yahoo.container.search.Fs4Config;
-import com.yahoo.document.GlobalId;
-import com.yahoo.fs4.BasicPacket;
-import com.yahoo.fs4.Packet;
-import com.yahoo.fs4.mplex.Backend;
-import com.yahoo.fs4.mplex.BackendTestCase;
-import com.yahoo.fs4.test.QueryTestCase;
import com.yahoo.language.simple.SimpleLinguistics;
-import com.yahoo.prelude.Ping;
-import com.yahoo.prelude.Pong;
import com.yahoo.prelude.fastsearch.ClusterParams;
import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
-import com.yahoo.prelude.fastsearch.FS4ResourcePool;
-import com.yahoo.prelude.fastsearch.FastHit;
import com.yahoo.prelude.fastsearch.FastSearcher;
import com.yahoo.prelude.fastsearch.SummaryParameters;
-import com.yahoo.prelude.fastsearch.test.fs4mock.MockBackend;
-import com.yahoo.prelude.fastsearch.test.fs4mock.MockFS4ResourcePool;
-import com.yahoo.prelude.fastsearch.test.fs4mock.MockFSChannel;
-import com.yahoo.processing.execution.Execution.Trace;
import com.yahoo.search.Query;
import com.yahoo.search.Result;
import com.yahoo.search.Searcher;
-import com.yahoo.search.dispatch.rpc.MockRpcResourcePoolBuilder;
import com.yahoo.search.dispatch.searchcluster.Node;
import com.yahoo.search.grouping.GroupingRequest;
import com.yahoo.search.grouping.request.AllOperation;
import com.yahoo.search.grouping.request.EachOperation;
import com.yahoo.search.grouping.request.GroupingOperation;
-import com.yahoo.search.query.SessionId;
import com.yahoo.search.rendering.RendererRegistry;
import com.yahoo.search.result.ErrorMessage;
import com.yahoo.search.searchchain.Execution;
-import com.yahoo.yolean.trace.TraceNode;
-import com.yahoo.yolean.trace.TraceVisitor;
import org.junit.Test;
-import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Level;
import java.util.logging.Logger;
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
+
/**
* Tests the Fast searcher
@@ -70,30 +40,12 @@ import static org.junit.Assert.assertTrue;
public class FastSearcherTestCase {
private final static DocumentdbInfoConfig documentdbInfoConfig = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder());
- private MockBackend mockBackend;
- @Test
- public void testNoNormalizing() {
- Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL);
- FastSearcher fastSearcher = new FastSearcher(new MockBackend(),
- new FS4ResourcePool("container.0", 1),
- MockDispatcher.create(Collections.emptyList()),
- new SummaryParameters(null),
- new ClusterParams("testhittype"),
- documentdbInfoConfig);
-
- MockFSChannel.setEmptyDocsums(false);
-
- Result result = doSearch(fastSearcher, new Query("?query=ignored"), 0, 10);
-
- assertTrue(result.hits().get(0).getRelevance().getScore() > 1000);
- }
@Test
public void testNullQuery() {
Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL);
- FastSearcher fastSearcher = new FastSearcher(new MockBackend(),
- new FS4ResourcePool("container.0", 1),
+ FastSearcher fastSearcher = new FastSearcher("container.0",
MockDispatcher.create(Collections.emptyList()),
new SummaryParameters(null),
new ClusterParams("testhittype"),
@@ -109,152 +61,6 @@ public class FastSearcherTestCase {
assertEquals(Error.NULL_QUERY.code, message.getCode());
}
- @Test
- public void testDispatchDotSummaries() {
- Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL);
- DocumentdbInfoConfig documentdbConfigWithOneDb =
- new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder()
- .name("testDb")
- .summaryclass(new DocumentdbInfoConfig.Documentdb.Summaryclass.Builder().name("simple").id(7))
- .rankprofile(new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder()
- .name("simpler").hasRankFeatures(false).hasSummaryFeatures(false))));
-
- List<Node> nodes = new ArrayList<>();
- nodes.add(new Node(0, "host1", 5000, 0));
- nodes.add(new Node(1, "host2", 5000, 0));
-
- var mockFs4ResourcePool = new MockFS4ResourcePool();
- var mockRpcResourcePool = new MockRpcResourcePoolBuilder().connection(0).connection(1).build();
-
- FastSearcher fastSearcher = new FastSearcher(new MockBackend(),
- mockFs4ResourcePool,
- MockDispatcher.create(nodes, mockFs4ResourcePool, mockRpcResourcePool, 1, new VipStatus()),
- new SummaryParameters(null),
- new ClusterParams("testhittype"),
- documentdbConfigWithOneDb);
-
- { // No direct.summaries
- String query = "?query=sddocname:a&summary=simple&timeout=20s";
- Result result = doSearch(fastSearcher, new Query(query), 0, 10);
- doFill(fastSearcher, result);
- ErrorMessage error = result.hits().getError();
- assertNull("Since we don't route to the dispatcher we hit the mock backend, so no error", error);
- }
-
- { // direct.summaries due to query cache
- String query = "?query=sddocname:a&ranking.queryCache&timeout=20s";
- Result result = doSearch(fastSearcher, new Query(query), 0, 10);
- doFill(fastSearcher, result);
- ErrorMessage error = result.hits().getError();
- assertEquals("Since we don't actually run summary backends we get this error when the Dispatcher is used",
- "getDocsums(..) attempted for node X", error.getDetailedMessage().replaceAll("\\d", "X"));
- }
-
- { // direct.summaries due to no summary features
- String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler&timeout=20s";
- Result result = doSearch(fastSearcher, new Query(query), 0, 10);
- doFill(fastSearcher, result);
- ErrorMessage error = result.hits().getError();
- assertEquals("Since we don't actually run summary backends we get this error when the Dispatcher is used",
- "getDocsums(..) attempted for node X", error.getDetailedMessage().replaceAll("\\d", "X"));
- }
- }
-
- @Test
- public void testQueryWithRestrict() {
- mockBackend = new MockBackend();
- DocumentdbInfoConfig documentdbConfigWithOneDb =
- new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("testDb")));
- FastSearcher fastSearcher = new FastSearcher(mockBackend,
- new FS4ResourcePool("container.0", 1),
- MockDispatcher.create(Collections.emptyList()),
- new SummaryParameters(null),
- new ClusterParams("testhittype"),
- documentdbConfigWithOneDb);
-
- Query query = new Query("?query=foo&model.restrict=testDb&groupingSessionCache=false");
- query.prepare();
- doSearch(fastSearcher, query, 0, 10);
-
- Packet receivedPacket = mockBackend.getChannel().getLastQueryPacket();
- byte[] encoded = QueryTestCase.packetToBytes(receivedPacket);
- byte[] correct = new byte[] {
- 0, 0, 0, 100, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, 6, 0, 10,
- QueryTestCase.ignored, QueryTestCase.ignored, QueryTestCase.ignored, QueryTestCase.ignored, // time left
- 0, 0, 0x40, 0x03, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 5, 109, 97, 116, 99, 104, 0, 0, 0, 1, 0, 0, 0, 24, 100, 111, 99, 117, 109, 101, 110, 116, 100, 98, 46, 115, 101, 97, 114, 99, 104, 100, 111, 99, 116, 121, 112, 101, 0, 0, 0, 6, 116, 101, 115, 116, 68, 98, 0, 0, 0, 1, 0, 0, 0, 7, 68, 1, 0, 3, 102, 111, 111
- };
- QueryTestCase.assertEqualArrays(correct, encoded);
- }
-
- @Test
- public void testSearch() {
- FastSearcher fastSearcher = createFastSearcher();
-
- Result result = doSearch(fastSearcher, new Query("?query=ignored"), 0, 10);
-
- Execution execution = new Execution(chainedAsSearchChain(fastSearcher), Execution.Context.createContextStub());
- assertEquals(2, result.getHitCount());
- execution.fill(result);
- assertCorrectHit1((FastHit)result.hits().get(0));
- assertCorrectTypes1((FastHit)result.hits().get(0));
- for (int idx = 0; idx < result.getHitCount(); idx++) {
- assertTrue(!result.hits().get(idx).isCached());
- }
-
- // Repeat the request a couple of times, to verify whether the packet cache works
- result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 10);
- assertEquals(2, result.getHitCount());
- execution.fill(result);
- assertCorrectHit1((FastHit) result.hits().get(0));
- for (int i = 0; i < result.getHitCount(); i++) {
- assertFalse(result.hits().get(i) + " should never be cached",
- result.hits().get(i).isCached());
- }
-
- result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 10);
- assertEquals(2, result.getHitCount());
- execution.fill(result);
- assertCorrectHit1((FastHit) result.hits().get(0));
- assertTrue("All hits are not cached", !result.isCached());
- for (int i = 0; i < result.getHitCount(); i++) {
- assertTrue(!result.hits().get(i).isCached());
- }
-
- // Test that partial result sets can be retrieved from the cache
- result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 1);
- assertEquals(1, result.getConcreteHitCount());
- execution.fill(result);
-
- result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 2);
- assertEquals(2, result.getConcreteHitCount());
- execution.fill(result);
- // No hit should be cached
- assertFalse(result.hits().get(0).isCached());
- assertFalse(result.hits().get(1).isCached());
-
- // Still nothing cached
- result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 2);
- assertEquals(2, result.getConcreteHitCount());
- execution.fill(result);
- // both first and second should now be cached
- assertFalse(result.hits().get(0).isCached());
- assertFalse(result.hits().get(1).isCached());
-
- // Tests that the cache _hit_ is not returned if _another_
- // hit is requested
-
- result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 1);
- assertEquals(1, result.getConcreteHitCount());
-
- result = doSearch(fastSearcher,new Query("?query=ignored"), 1, 1);
- assertEquals(1, result.getConcreteHitCount());
-
- for (int i = 0; i < result.getHitCount(); i++) {
- assertFalse("Hit " + i + " should not be cached.",
- result.hits().get(i).isCached());
- }
- }
-
private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) {
List<Searcher> searchers = new ArrayList<>();
searchers.add(topOfChain);
@@ -272,79 +78,9 @@ public class FastSearcherTestCase {
return new Execution(chainedAsSearchChain(searcher), context);
}
- private void doFill(Searcher searcher, Result result) {
- createExecution(searcher).fill(result);
- }
-
- @Test
- public void testThatPropertiesAreReencoded() throws Exception {
- FastSearcher fastSearcher = createFastSearcher();
-
- Query query = new Query("?query=ignored&dispatch.summaries=false&groupingSessionCache=false");
- query.getRanking().setQueryCache(true);
- Result result = doSearch(fastSearcher, query, 0, 10);
-
- Execution execution = new Execution(chainedAsSearchChain(fastSearcher), Execution.Context.createContextStub());
- assertEquals(2, result.getHitCount());
- execution.fill(result);
-
- BasicPacket receivedPacket = mockBackend.getChannel().getLastReceived();
- ByteBuffer buf = ByteBuffer.allocate(1000);
- receivedPacket.encode(buf);
- buf.flip();
- byte[] actual = new byte[buf.remaining()];
- buf.get(actual);
-
- SessionId sessionId = query.getSessionId();
- byte IGNORE = 69;
- ByteBuffer answer = ByteBuffer.allocate(1024);
- answer.put(new byte[] { 0, 0, 0, (byte)(141+sessionId.asUtf8String().getByteLength()), 0, 0, 0, -37, 0, 0, 16, 17, 0, 0, 0, 0,
- // query timeout
- IGNORE, IGNORE, IGNORE, IGNORE,
- // "default" - rank profile
- 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 0x03,
- // 3 property entries (rank, match, caches)
- 0, 0, 0, 3,
- // rank: sessionId => qrserver.0.XXXXXXXXXXXXX.0
- 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 1, 0, 0, 0, 9, 's', 'e', 's', 's', 'i', 'o', 'n', 'I', 'd'});
- answer.putInt(sessionId.asUtf8String().getBytes().length);
- answer.put(sessionId.asUtf8String().getBytes());
- answer.put(new byte [] {
- // match: documentdb.searchdoctype => test
- 0, 0, 0, 5, 'm', 'a', 't', 'c', 'h', 0, 0, 0, 1, 0, 0, 0, 24, 'd', 'o', 'c', 'u', 'm', 'e', 'n', 't', 'd', 'b', '.', 's', 'e', 'a', 'r', 'c', 'h', 'd', 'o', 'c', 't', 'y', 'p', 'e', 0, 0, 0, 4, 't', 'e', 's', 't',
- // sessionId => qrserver.0.XXXXXXXXXXXXX.0
- 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e'});
- byte [] expected = new byte [answer.position()];
- answer.flip();
- answer.get(expected);
-
- for (int i = 0; i < expected.length; ++i) {
- if (expected[i] == IGNORE) {
- actual[i] = IGNORE;
- }
- }
- assertArrayEquals(expected, actual);
- }
-
- private FastSearcher createFastSearcher() {
- mockBackend = new MockBackend();
- ConfigGetter<DocumentdbInfoConfig> getter = new ConfigGetter<>(DocumentdbInfoConfig.class);
- DocumentdbInfoConfig config = getter.getConfig("file:src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg");
-
- MockFSChannel.resetDocstamp();
- Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL);
- return new FastSearcher(mockBackend,
- new FS4ResourcePool("container.0", 1),
- MockDispatcher.create(Collections.emptyList()),
- new SummaryParameters(null),
- new ClusterParams("testhittype"),
- config);
- }
-
@Test
public void testSinglePassGroupingIsForcedWithSingleNodeGroups() {
- FastSearcher fastSearcher = new FastSearcher(new MockBackend(),
- new FS4ResourcePool("container.0", 1),
+ FastSearcher fastSearcher = new FastSearcher("container.0",
MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 123, 0))),
new SummaryParameters(null),
new ClusterParams("testhittype"),
@@ -368,8 +104,7 @@ public class FastSearcherTestCase {
public void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() {
MockDispatcher dispatcher = MockDispatcher.create(ImmutableList.of(new Node(0, "host0", 123, 0), new Node(2, "host1", 123, 0)));
- FastSearcher fastSearcher = new FastSearcher(new MockBackend(),
- new FS4ResourcePool("container.0", 1),
+ FastSearcher fastSearcher = new FastSearcher("container.0",
dispatcher,
new SummaryParameters(null),
new ClusterParams("testhittype"),
@@ -401,95 +136,4 @@ public class FastSearcherTestCase {
assertForceSinglePassIs(expected, child);
}
- @Test
- public void testPing() throws IOException, InterruptedException {
- Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL);
- BackendTestCase.MockServer server = new BackendTestCase.MockServer();
- FS4ResourcePool listeners = new FS4ResourcePool(new Fs4Config(new Fs4Config.Builder()), new QrConfig(new QrConfig.Builder()));
- Backend backend = listeners.getBackend(server.host.getHostString(),server.host.getPort());
- FastSearcher fastSearcher = new FastSearcher(backend,
- new FS4ResourcePool("container.0", 1),
- MockDispatcher.create(Collections.emptyList()),
- new SummaryParameters(null),
- new ClusterParams("testhittype"),
- documentdbInfoConfig);
- server.dispatch.packetData = BackendTestCase.PONG;
- server.dispatch.setNoChannel();
- Chain<Searcher> chain = new Chain<>(fastSearcher);
- Execution e = new Execution(chain, Execution.Context.createContextStub());
- Pong pong = e.ping(new Ping());
- backend.shutdown();
- server.dispatch.socket.close();
- server.dispatch.connection.close();
- server.worker.join();
- pong.setPingInfo("blbl");
- assertEquals("Result of pinging using blbl", pong.toString());
- }
-
- private void assertCorrectTypes1(FastHit hit) {
- assertEquals(String.class, hit.getField("TITLE").getClass());
- assertEquals(Integer.class, hit.getField("BYTES").getClass());
- }
-
- private void assertCorrectHit1(FastHit hit) {
- assertEquals(
- "StudyOfMadonna.com - Interviews, Articles, Reviews, Quotes, Essays and more..",
- hit.getField("TITLE"));
- assertEquals("352", hit.getField("WORDS").toString());
- assertEquals(2003., hit.getRelevance().getScore(), 0.01d);
- assertEquals("index:testhittype/234/" + asHexString(hit.getGlobalId()), hit.getId().toString());
- assertEquals("9190", hit.getField("BYTES").toString());
- assertEquals("testhittype", hit.getSource());
- }
-
- private static String asHexString(GlobalId gid) {
- StringBuilder sb = new StringBuilder();
- byte[] rawGid = gid.getRawId();
- for (byte b : rawGid) {
- String hex = Integer.toHexString(0xFF & b);
- if (hex.length() == 1)
- sb.append('0');
- sb.append(hex);
- }
- return sb.toString();
- }
-
- @Test
- public void null_summary_is_included_in_trace() {
- String summary = null;
- assertThat(getTraceString(summary), containsString("summary=[null]"));
- }
-
- @Test
- public void non_null_summary_is_included_in_trace() {
- String summary = "all";
- assertThat(getTraceString(summary), containsString("summary='all'"));
- }
-
- private String getTraceString(String summary) {
- FastSearcher fastSearcher = createFastSearcher();
-
- Query query = new Query("?query=ignored");
- query.getPresentation().setSummary(summary);
- query.setTraceLevel(2);
-
- Result result = doSearch(fastSearcher, query, 0, 10);
- doFill(fastSearcher, result);
-
- Trace trace = query.getContext(false).getTrace();
- final AtomicReference<String> fillTraceString = new AtomicReference<>();
-
-
- trace.traceNode().accept(new TraceVisitor() {
- @Override
- public void visit(TraceNode traceNode) {
- if (traceNode.payload() instanceof String && traceNode.payload().toString().contains("fill to dispatch"))
- fillTraceString.set((String) traceNode.payload());
-
- }
- });
-
- return fillTraceString.get();
- }
-
}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java
deleted file mode 100644
index 6eab16045c2..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java
+++ /dev/null
@@ -1,127 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.fastsearch.test;
-
-import com.google.common.util.concurrent.MoreExecutors;
-import com.yahoo.container.QrSearchersConfig;
-import com.yahoo.container.handler.VipStatus;
-import com.yahoo.net.HostName;
-import com.yahoo.prelude.fastsearch.ClusterParams;
-import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig;
-import com.yahoo.prelude.fastsearch.FastSearcher;
-import com.yahoo.prelude.fastsearch.SummaryParameters;
-import com.yahoo.prelude.fastsearch.test.fs4mock.MockBackend;
-import com.yahoo.prelude.fastsearch.test.fs4mock.MockFS4ResourcePool;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.dispatch.rpc.MockRpcResourcePoolBuilder;
-import com.yahoo.search.dispatch.rpc.RpcResourcePool;
-import com.yahoo.search.dispatch.searchcluster.Node;
-import com.yahoo.search.searchchain.Execution;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * @author bratseth
- */
-class FastSearcherTester {
-
- public static final String selfHostname = HostName.getLocalhost();
-
- private final MockFS4ResourcePool mockFS4ResourcePool;
- private final RpcResourcePool mockRpcResourcePool;
- private final FastSearcher fastSearcher;
- private final MockDispatcher mockDispatcher;
- private final VipStatus vipStatus;
-
- public FastSearcherTester(int containerClusterSize, Node searchNode) {
- this(containerClusterSize, Collections.singletonList(searchNode));
- }
-
- public FastSearcherTester(int containerClusterSize, String... hostAndPortAndGroupStrings) {
- this(containerClusterSize, toNodes(hostAndPortAndGroupStrings));
- }
-
- public FastSearcherTester(int containerClusterSize, List<Node> searchNodes) {
- String clusterId = "a";
-
- var b = new QrSearchersConfig.Builder();
- var searchClusterB = new QrSearchersConfig.Searchcluster.Builder();
- searchClusterB.name(clusterId);
- b.searchcluster(searchClusterB);
- vipStatus = new VipStatus(b.build());
-
- mockFS4ResourcePool = new MockFS4ResourcePool();
- var builder = new MockRpcResourcePoolBuilder();
- searchNodes.forEach(node -> builder.connection(node.key()));
- mockRpcResourcePool = builder.build();
- mockDispatcher = MockDispatcher.create(searchNodes, mockFS4ResourcePool, mockRpcResourcePool, containerClusterSize, vipStatus);
- fastSearcher = new FastSearcher(new MockBackend(selfHostname, 0L, true),
- mockFS4ResourcePool,
- mockDispatcher,
- new SummaryParameters(null),
- new ClusterParams("testhittype"),
- new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder()));
- }
-
- private static List<Node> toNodes(String... hostAndPortAndGroupStrings) {
- List<Node> nodes = new ArrayList<>();
- int key = 0;
- for (String s : hostAndPortAndGroupStrings) {
- String[] parts = s.split(":");
- nodes.add(new Node(key++, parts[0], Integer.parseInt(parts[1]), Integer.parseInt(parts[2])));
- }
- return nodes;
- }
-
- public Result search(String query) {
- Result result = fastSearcher.search(new Query(query), new Execution(Execution.Context.createContextStub()));
- assertEquals(null, result.hits().getError());
- return result;
- }
-
- /** Returns the number of times a backend for this hostname and port has been requested */
- public int requestCount(String hostname, int port) {
- return mockFS4ResourcePool.requestCount(hostname, port);
- }
-
- public MockDispatcher dispatcher() { return mockDispatcher; }
-
- /** Sets the response status of a node and ping it to update the monitor status */
- public void setResponding(String hostname, boolean responding) {
- // Start/stop returning a failing backend
- mockFS4ResourcePool.setResponding(hostname, responding);
-
- // Make the search cluster monitor notice right now in this thread
- Node node = mockDispatcher.searchCluster().nodesByHost().get(hostname).iterator().next();
- mockDispatcher.searchCluster().ping(node, MoreExecutors.directExecutor());
- }
-
- /** Sets the response status of a node and ping it to update the monitor status */
- public void setActiveDocuments(String hostname, long activeDocuments) {
- mockFS4ResourcePool.setActiveDocuments(hostname, activeDocuments);
-
- // Make the search cluster monitor notice right now in this thread
- Node node = mockDispatcher.searchCluster().nodesByHost().get(hostname).iterator().next();
- mockDispatcher.searchCluster().ping(node, MoreExecutors.directExecutor());
- mockDispatcher.searchCluster().pingIterationCompleted();
- }
-
- public VipStatus vipStatus() { return vipStatus; }
-
- /** Retrying is needed because earlier pings from the monitoring thread may interfere with the testing thread */
- public void waitForInRotationIs(boolean expectedRotationStatus) {
- int triesLeft = 9000;
- while (vipStatus.isInRotation() != expectedRotationStatus && triesLeft > 0) {
- triesLeft--;
- try { Thread.sleep(10); } catch (InterruptedException e) {}
- }
- if (triesLeft == 0)
- fail("Did not reach VIP in rotation status = " + expectedRotationStatus + " after trying for 90 seconds");
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
index ccb265b799b..afb9cf6f571 100644
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
+++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java
@@ -2,8 +2,6 @@
package com.yahoo.prelude.fastsearch.test;
import com.yahoo.container.handler.VipStatus;
-import com.yahoo.prelude.fastsearch.FS4PingFactory;
-import com.yahoo.prelude.fastsearch.FS4ResourcePool;
import com.yahoo.search.dispatch.Dispatcher;
import com.yahoo.search.dispatch.rpc.RpcInvokerFactory;
import com.yahoo.search.dispatch.rpc.RpcResourcePool;
@@ -15,23 +13,24 @@ import java.util.List;
class MockDispatcher extends Dispatcher {
public static MockDispatcher create(List<Node> nodes) {
- var fs4ResourcePool = new FS4ResourcePool("container.0", 1);
var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes));
- return create(nodes, fs4ResourcePool, rpcResourcePool, 1, new VipStatus());
+ return create(nodes, rpcResourcePool, 1, new VipStatus());
}
- public static MockDispatcher create(List<Node> nodes, FS4ResourcePool fs4ResourcePool, RpcResourcePool rpcResourcePool,
+ public static MockDispatcher create(List<Node> nodes, RpcResourcePool rpcResourcePool,
int containerClusterSize, VipStatus vipStatus) {
var dispatchConfig = toDispatchConfig(nodes);
var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus);
- return new MockDispatcher(searchCluster, dispatchConfig, fs4ResourcePool, rpcResourcePool);
+ return new MockDispatcher(searchCluster, dispatchConfig, rpcResourcePool);
}
- private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool,
- RpcResourcePool rpcResourcePool) {
- super(searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster, dispatchConfig.dispatchWithProtobuf()),
- new FS4PingFactory(fs4ResourcePool), new MockMetric());
+ private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) {
+ this(searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster, dispatchConfig.dispatchWithProtobuf()));
+ }
+
+ private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) {
+ super(searchCluster, dispatchConfig, invokerFactory, invokerFactory, new MockMetric());
}
private static DispatchConfig toDispatchConfig(List<Node> nodes) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/DispatchThread.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/DispatchThread.java
deleted file mode 100644
index d09e8856ee7..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/DispatchThread.java
+++ /dev/null
@@ -1,101 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-// -*- mode: java; folded-file: t; c-basic-offset: 4 -*-
-//
-//
-package com.yahoo.prelude.fastsearch.test.fs4mock;
-
-
-import com.yahoo.prelude.ConfigurationException;
-
-
-/**
- * Thread-wrapper for MockFDispatch
- *
- * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a>
- */
-public class DispatchThread extends Thread {
- int listenPort;
- long replyDelay;
- long byteDelay;
- MockFDispatch dispatch;
- Object barrier = new Object();
-
- /**
- * Instantiate MockFDispatch; if the wanted port is taken we
- * bump the port number. Note that the delays are not
- * accurate: in reality they will be significantly longer for
- * low values.
- *
- * @param listenPort Wanted port number, note that this may be
- * bumped if someone is already running something
- * on this port, so it is a starting point for
- * scanning only
- * @param replyDelay how many milliseconds we should delay when
- * replying
- * @param byteDelay how many milliseconds we delay for each byte
- * written
- */
-
- public DispatchThread(int listenPort, long replyDelay, long byteDelay) {
- this.listenPort = listenPort;
- this.replyDelay = replyDelay;
- this.byteDelay = byteDelay;
- dispatch = new MockFDispatch(listenPort, replyDelay, byteDelay);
- dispatch.setBarrier(barrier);
- }
-
- /**
- * Run the MockFDispatch and anticipate multiple instances of
- * same running.
- */
- public void run() {
- int maxTries = 20;
- // the following section is here to make sure that this
- // test is somewhat robust, ie. if someone is already
- // listening to the port in question, we'd like to NOT
- // fail, but keep probing until we find a port we can use.
- boolean up = false;
-
- while ((!up) && (maxTries-- != 0)) {
- try {
- dispatch.run();
- up = true;
- } catch (ConfigurationException e) {
- listenPort++;
- dispatch.setListenPort(listenPort);
- }
- }
- }
-
- /**
- * Wait until MockFDispatch is ready to accept connections
- * or we time out and indicate which of the two outcomes it was.
- *
- * @return If we time out we return <code>false</code>. Else we
- * return <code>true</code>
- *
- */
- public boolean waitOnBarrier(long timeout) throws InterruptedException {
- long start = System.currentTimeMillis();
-
- synchronized (barrier) {
- barrier.wait(timeout);
- }
- long diff = System.currentTimeMillis() - start;
-
- return (diff < timeout);
- }
-
- /**
- * Return the port on which the MockFDispatch actually listens.
- * use this instead of assuming where it is since, if more than
- * one application tries to use the port we've assigned to it
- * we might have to up the port number.
- *
- * @return port number of active MockFDispatch instance
- *
- */
- public int listenPort() {
- return listenPort;
- }
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockBackend.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockBackend.java
deleted file mode 100644
index d3fbf8f3645..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockBackend.java
+++ /dev/null
@@ -1,53 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.fastsearch.test.fs4mock;
-
-import com.yahoo.fs4.mplex.Backend;
-import com.yahoo.fs4.mplex.FS4Channel;
-
-/**
- * @author bratseth
- */
-public class MockBackend extends Backend {
-
- private String hostname;
- private final long activeDocumentsInBackend;
- private final boolean working;
-
- /** Created lazily as we want to have just one but it depends on the channel */
- private MockFSChannel channel = null;
-
- public MockBackend() {
- this("", 0L, true);
- }
-
- public MockBackend(String hostname, long activeDocumentsInBackend, boolean working) {
- super();
- this.hostname = hostname;
- this.activeDocumentsInBackend = activeDocumentsInBackend;
- this.working = working;
- }
-
- @Override
- public FS4Channel openChannel() {
- if (channel == null)
- channel = working ? new MockFSChannel(activeDocumentsInBackend, this)
- : new NonWorkingMockFSChannel(this);
- return channel;
- }
-
- @Override
- public FS4Channel openPingChannel() { return openChannel(); }
-
- @Override
- public String getHost() { return hostname; }
-
- /** Returns the channel in use or null if no channel has been used yet */
- public MockFSChannel getChannel() { return channel; }
-
- public void shutdown() {}
-
- @Override
- public boolean probeConnection() {
- return working;
- }
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFDispatch.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFDispatch.java
deleted file mode 100644
index 6956f288d1a..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFDispatch.java
+++ /dev/null
@@ -1,212 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.fastsearch.test.fs4mock;
-
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.nio.ByteBuffer;
-import java.nio.channels.ClosedByInterruptException;
-import java.nio.channels.ClosedChannelException;
-import java.nio.channels.ServerSocketChannel;
-import java.nio.channels.SocketChannel;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import com.yahoo.prelude.ConfigurationException;
-import com.yahoo.prelude.fastsearch.test.DocsumDefinitionTestCase;
-
-
-/**
- * A server which replies to any query with the same query result after
- * a configurable delay, with a configurable slowness (delay between each byte).
- * Connections are never timed out.
- *
- * @author bratseth
- */
-public class MockFDispatch {
-
- private static int connectionCount = 0;
-
- private static Logger log = Logger.getLogger(MockFDispatch.class.getName());
-
- /** The port we accept incoming requests at */
- private int listenPort = 0;
-
- private long replyDelay;
-
- private long byteDelay;
-
- private Object barrier;
-
- private static byte[] queryResultPacketData = new byte[] {
- 0, 0, 0, 64, 0, 0,
- 0, 214 - 256, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0,
- 25, 0, 0, 0, 111, 0, 0, 0, 97, 0, 0, 0, 3, 0, 0, 0, 23, 0, 0, 0, 7, 0, 0,
- 0, 36, 0, 0, 0, 4, 0, 0, 0, 21, 0, 0, 0, 8, 0, 0, 0, 37};
-
- private static byte[] docsumData = DocsumDefinitionTestCase.makeDocsum();
-
- private static byte[] docsumHeadPacketData = new byte[] {
- 0, 0, 3, 39, 0, 0,
- 0, 205 - 256, 0, 0, 0, 1, 0, 0, 0, 0};
-
- private static byte[] eolPacketData = new byte[] {
- 0, 0, 0, 8, 0, 0, 0,
- 200 - 256, 0, 0, 0, 1 };
-
- private Set<ConnectionThread> connectionThreads = new HashSet<>();
-
- public MockFDispatch(int listenPort, long replyDelay, long byteDelay) {
- this.replyDelay = replyDelay;
- this.byteDelay = byteDelay;
- this.listenPort = listenPort;
- }
-
- public void setBarrier(Object barrier) {
- this.barrier = barrier;
- }
-
- public void setListenPort(int listenPort) {
- this.listenPort = listenPort;
- }
-
- public void run() {
- try {
- ServerSocketChannel channel = createServerSocket(listenPort);
-
- channel.socket().setReuseAddress(true);
- while (!Thread.currentThread().isInterrupted()) {
- try {
- // notify those waiting at the barrier that they
- // can now proceed and talk to us
- synchronized (barrier) {
- if (barrier != null) {
- barrier.notify();
- }
- }
- SocketChannel socketChannel = channel.accept();
-
- connectionThreads.add(new ConnectionThread(socketChannel));
- } catch (ClosedByInterruptException e) {// We'll exit
- } catch (ClosedChannelException e) {
- return;
- } catch (Exception e) {
- log.log(Level.WARNING, "Unexpected error reading request", e);
- }
- }
- channel.close();
- } catch (IOException e) {
- throw new ConfigurationException("Socket channel failure", e);
- }
- }
-
- private ServerSocketChannel createServerSocket(int listenPort)
- throws IOException {
- ServerSocketChannel channel = ServerSocketChannel.open();
- ServerSocket socket = channel.socket();
-
- socket.bind(
- new InetSocketAddress(InetAddress.getLocalHost(), listenPort));
- String host = socket.getInetAddress().getHostName();
-
- log.fine("Accepting dfispatch requests at " + host + ":" + listenPort);
- return channel;
- }
-
- public static void main(String[] args) {
- log.setLevel(Level.FINE);
- MockFDispatch m = new MockFDispatch(7890, Integer.parseInt(args[0]),
- Integer.parseInt(args[1]));
-
- m.run();
- }
-
- private class ConnectionThread extends Thread {
-
- private ByteBuffer writeBuffer = ByteBuffer.allocate(2000);
-
- private ByteBuffer readBuffer = ByteBuffer.allocate(2000);
-
- private int connectionNr = 0;
-
- private SocketChannel channel;
-
- public ConnectionThread(SocketChannel channel) {
- this.channel = channel;
- fillBuffer(writeBuffer);
- start();
- }
-
- private void fillBuffer(ByteBuffer buffer) {
- buffer.clear();
- buffer.put(queryResultPacketData);
- buffer.put(docsumHeadPacketData);
- buffer.put(docsumData);
- buffer.put(docsumHeadPacketData);
- buffer.put(docsumData);
- buffer.put(eolPacketData);
- }
-
- public void run() {
- connectionNr = connectionCount++;
- log.fine("Opened connection " + connectionNr);
-
- try {
- long lastRequest = System.currentTimeMillis();
-
- while ((System.currentTimeMillis() - lastRequest) <= 5000
- && (!isInterrupted())) {
- readBuffer.clear();
- channel.read(readBuffer);
- lastRequest = System.currentTimeMillis();
- delay(replyDelay);
-
- if (byteDelay > 0) {
- writeSlow(writeBuffer);
- } else {
- write(writeBuffer);
- }
- log.fine(
- "Replied in "
- + (System.currentTimeMillis() - lastRequest)
- + " ms");
- }
-
- log.fine("Closing timed out connection " + connectionNr);
- connectionCount--;
- channel.close();
- } catch (IOException e) {}
- }
-
- private void write(ByteBuffer writeBuffer) throws IOException {
- writeBuffer.flip();
- channel.write(writeBuffer);
- }
-
- private void writeSlow(ByteBuffer writeBuffer) throws IOException {
- writeBuffer.flip();
- int dataSize = writeBuffer.limit();
-
- for (int i = 0; i < dataSize; i++) {
- writeBuffer.position(i);
- writeBuffer.limit(i + 1);
- channel.write(writeBuffer);
- delay(byteDelay);
- }
- writeBuffer.limit(dataSize);
- }
-
- private void delay(long delay) {
-
- try {
- Thread.sleep(delay);
- } catch (InterruptedException e) {}
- }
-
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFS4ResourcePool.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFS4ResourcePool.java
deleted file mode 100644
index 0d756cbeff3..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFS4ResourcePool.java
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.fastsearch.test.fs4mock;
-
-import com.yahoo.fs4.mplex.Backend;
-import com.yahoo.prelude.fastsearch.FS4ResourcePool;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * @author bratseth
- */
-public class MockFS4ResourcePool extends FS4ResourcePool {
-
- private final Map<String, Integer> requestsPerBackend = new HashMap<>();
- private final Set<String> nonRespondingBackends = new HashSet<>();
- private final Map<String, Long> activeDocumentsInBackend = new HashMap<>();
- private final long testingThreadId;
-
- public MockFS4ResourcePool() {
- super("container.0", 1);
- this.testingThreadId = Thread.currentThread().getId();
- }
-
- @Override
- public Backend getBackend(String hostname, int port) {
- countRequest(hostname + ":" + port);
- if (nonRespondingBackends.contains(hostname))
- return new MockBackend(hostname, 0L, false);
- else
- return new MockBackend(hostname, activeDocumentsInBackend.getOrDefault(hostname, 0L), true);
- }
-
- /**
- * Returns the number of times a backend for this hostname and port has been requested
- * from the thread creating this
- */
- public int requestCount(String hostname, int port) {
- return requestsPerBackend.getOrDefault(hostname + ":" + port, 0);
- }
-
- /** Sets the number of active documents the given host will report to have in ping responses */
- public void setActiveDocuments(String hostname, long activeDocuments) {
- activeDocumentsInBackend.put(hostname, activeDocuments);
- }
-
- private void countRequest(String hostAndPort) {
- // ignore requests from the ping thread to avoid timing issues
- if (Thread.currentThread().getId() != testingThreadId) return;
-
- requestsPerBackend.put(hostAndPort, requestsPerBackend.getOrDefault(hostAndPort, 0) + 1);
- }
-
- public void setResponding(String hostname, boolean responding) {
- if (responding)
- nonRespondingBackends.remove(hostname);
- else
- nonRespondingBackends.add(hostname);
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java
deleted file mode 100644
index db14a2894db..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java
+++ /dev/null
@@ -1,176 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.fastsearch.test.fs4mock;
-
-import com.yahoo.document.GlobalId;
-import com.yahoo.fs4.BasicPacket;
-import com.yahoo.fs4.BufferTooSmallException;
-import com.yahoo.fs4.DocumentInfo;
-import com.yahoo.fs4.EolPacket;
-import com.yahoo.fs4.GetDocSumsPacket;
-import com.yahoo.fs4.Packet;
-import com.yahoo.fs4.PacketDecoder;
-import com.yahoo.fs4.PingPacket;
-import com.yahoo.fs4.PongPacket;
-import com.yahoo.fs4.QueryPacket;
-import com.yahoo.fs4.QueryResultPacket;
-import com.yahoo.fs4.mplex.Backend;
-import com.yahoo.fs4.mplex.FS4Channel;
-import com.yahoo.prelude.fastsearch.test.DocsumDefinitionTestCase;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-
-/**
- * A channel which returns hardcoded packets of the same type as fdispatch
- */
-public class MockFSChannel extends FS4Channel {
-
- /** The number of active documents this should report in ping reponses */
- private final long activeDocuments;
-
- MockFSChannel(Backend backend) {
- this(0, backend);
- }
-
- MockFSChannel(long activeDocuments, Backend backend) {
- super(backend, 0);
- this.activeDocuments = activeDocuments;
- }
-
- private BasicPacket lastReceived = null;
-
- private static QueryPacket lastQueryPacket = null;
-
- /** Initial value of docstamp */
- private static int docstamp = 1088490666;
-
- private static boolean emptyDocsums = false;
-
- @Override
- public synchronized boolean sendPacket(BasicPacket packet) {
- try {
- if (packet instanceof Packet)
- packet.encode(ByteBuffer.allocate(65536), 0);
- } catch (BufferTooSmallException e) {
- throw new RuntimeException("Too small buffer to encode packet in mock backend.");
- }
-
- if (packet instanceof QueryPacket)
- lastQueryPacket = (QueryPacket) packet;
-
- lastReceived = packet;
- notifyMonitor();
- return true;
- }
-
- /** Change docstamp to invalidate cache */
- public static void resetDocstamp() {
- docstamp = 1088490666;
- }
-
- /** Flip sending (in)valid docsums */
- public static void setEmptyDocsums(boolean d) {
- emptyDocsums = d;
- }
-
- /** Returns the last query packet received or null if none */
- public QueryPacket getLastQueryPacket() {
- return lastQueryPacket;
- }
-
- public BasicPacket getLastReceived() {
- return lastReceived;
- }
-
- public BasicPacket[] receivePackets(long timeout, int packetCount) {
- List<BasicPacket> packets = new java.util.ArrayList<>();
-
- if (lastReceived instanceof QueryPacket) {
- lastQueryPacket = (QueryPacket) lastReceived;
- QueryResultPacket result = QueryResultPacket.create();
-
- result.setDocstamp(docstamp);
- result.setChannel(0);
- result.setTotalDocumentCount(2);
- result.setOffset(lastQueryPacket.getOffset());
-
- if (lastQueryPacket.getOffset() == 0
- && lastQueryPacket.getLastOffset() >= 1) {
- result.addDocument(
- new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(123),
- 2003, 234, 0));
- }
- if (lastQueryPacket.getOffset() <= 1
- && lastQueryPacket.getLastOffset() >= 2) {
- result.addDocument(
- new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(456),
- 1855, 234, 1));
- }
- packets.add(result);
- }
- else if (lastReceived instanceof GetDocSumsPacket) {
- addDocsums(packets, lastQueryPacket);
- }
- else if (lastReceived instanceof PingPacket) {
- packets.add(new PongPacket(activeDocuments));
- }
- while (packetCount >= 0 && packets.size() > packetCount) {
- packets.remove(packets.size() - 1);
- }
-
- return packets.toArray(new BasicPacket[packets.size()]);
- }
-
- /** Adds the number of docsums requested in queryPacket.getHits() */
- private void addDocsums(List<BasicPacket> packets, QueryPacket queryPacket) {
- int numHits = queryPacket.getHits();
-
- if (lastReceived instanceof GetDocSumsPacket) {
- numHits = ((GetDocSumsPacket) lastReceived).getNumDocsums();
- }
- for (int i = 0; i < numHits; i++) {
- ByteBuffer buffer;
-
- if (emptyDocsums) {
- buffer = createEmptyDocsumPacketData();
- } else {
- int[] docids = {
- 123, 456, 789, 789, 789, 789, 789, 789, 789,
- 789, 789, 789 };
-
- buffer = createDocsumPacketData(docids[i], DocsumDefinitionTestCase.makeDocsum());
- }
- buffer.position(0);
- packets.add(PacketDecoder.decode(buffer));
- }
- packets.add(EolPacket.create());
- }
-
- private ByteBuffer createEmptyDocsumPacketData() {
- ByteBuffer buffer = ByteBuffer.allocate(16);
-
- buffer.limit(buffer.capacity());
- buffer.position(0);
- buffer.putInt(12); // length
- buffer.putInt(205); // a code for docsumpacket
- buffer.putInt(0); // channel
- buffer.putInt(0); // dummy location
- return buffer;
- }
-
- private ByteBuffer createDocsumPacketData(int docid, byte[] docsumData) {
- ByteBuffer buffer = ByteBuffer.allocate(docsumData.length + 4 + 8 + GlobalId.LENGTH);
-
- buffer.limit(buffer.capacity());
- buffer.position(0);
- buffer.putInt(docsumData.length + 8 + GlobalId.LENGTH);
- buffer.putInt(205); // Docsum packet code
- buffer.putInt(0);
- byte[] rawGid = DocsumDefinitionTestCase.createGlobalId(docid).getRawId();
- buffer.put(rawGid);
- buffer.put(docsumData);
- return buffer;
- }
-
- public void close() {}
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/NonWorkingMockFSChannel.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/NonWorkingMockFSChannel.java
deleted file mode 100644
index c7425afd611..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/NonWorkingMockFSChannel.java
+++ /dev/null
@@ -1,21 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.fastsearch.test.fs4mock;
-
-import com.yahoo.fs4.BasicPacket;
-import com.yahoo.fs4.mplex.Backend;
-
-/**
- * @author bratseth
- */
-public class NonWorkingMockFSChannel extends MockFSChannel {
-
- public NonWorkingMockFSChannel(Backend backend) {
- super(backend);
- }
-
- @Override
- public synchronized boolean sendPacket(BasicPacket bPacket) {
- return false;
- }
-
-}