From 250f7e16efa36fe2b5e24bf75fe3f0af5cb4fed8 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Wed, 18 Sep 2019 10:47:22 +0200 Subject: Revert "Revert "Revert "Revert "Balder/no more fs4 dispatching from fastsearcher"""" --- .../com/yahoo/fs4/PacketQueryTracerTestCase.java | 110 ------- .../java/com/yahoo/fs4/mplex/BackendTestCase.java | 208 ------------ .../yahoo/fs4/test/GetDocSumsPacketTestCase.java | 107 ------ .../yahoo/fs4/test/HexByteIteratorTestCase.java | 42 --- .../com/yahoo/fs4/test/PacketDecoderTestCase.java | 186 ----------- .../java/com/yahoo/fs4/test/PacketTestCase.java | 229 ------------- .../com/yahoo/fs4/test/QueryResultTestCase.java | 113 ------- .../java/com/yahoo/fs4/test/QueryTestCase.java | 319 ------------------ .../com/yahoo/fs4/test/RankFeaturesTestCase.java | 136 -------- .../prelude/cluster/ClusterSearcherTestCase.java | 67 ++-- .../fastsearch/FS4SearchInvokerTestCase.java | 72 ---- .../fastsearch/test/DirectSearchTestCase.java | 137 -------- .../fastsearch/test/FastSearcherTestCase.java | 364 +-------------------- .../fastsearch/test/FastSearcherTester.java | 127 ------- .../prelude/fastsearch/test/MockDispatcher.java | 19 +- .../fastsearch/test/fs4mock/DispatchThread.java | 101 ------ .../fastsearch/test/fs4mock/MockBackend.java | 53 --- .../fastsearch/test/fs4mock/MockFDispatch.java | 212 ------------ .../test/fs4mock/MockFS4ResourcePool.java | 63 ---- .../fastsearch/test/fs4mock/MockFSChannel.java | 176 ---------- .../test/fs4mock/NonWorkingMockFSChannel.java | 21 -- .../yahoo/search/dispatch/MockSearchCluster.java | 5 - .../grouping/vespa/HitConverterTestCase.java | 14 - .../search/query/test/RankFeaturesTestCase.java | 140 ++++++++ 24 files changed, 184 insertions(+), 2837 deletions(-) delete mode 100644 container-search/src/test/java/com/yahoo/fs4/PacketQueryTracerTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/fs4/test/HexByteIteratorTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/fs4/test/PacketDecoderTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/fs4/test/PacketTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/fs4/test/QueryResultTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/fs4/test/QueryTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/fs4/test/RankFeaturesTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/FS4SearchInvokerTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/DispatchThread.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockBackend.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFDispatch.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFS4ResourcePool.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java delete mode 100644 container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/NonWorkingMockFSChannel.java create mode 100644 container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java (limited to 'container-search/src/test') diff --git a/container-search/src/test/java/com/yahoo/fs4/PacketQueryTracerTestCase.java b/container-search/src/test/java/com/yahoo/fs4/PacketQueryTracerTestCase.java deleted file mode 100644 index 20d9b61c177..00000000000 --- a/container-search/src/test/java/com/yahoo/fs4/PacketQueryTracerTestCase.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.fs4; - -import static org.junit.Assert.*; - -import java.io.IOException; -import java.io.StringWriter; -import java.nio.ByteBuffer; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.yahoo.fs4.mplex.FS4Channel; -import com.yahoo.fs4.mplex.InvalidChannelException; -import com.yahoo.search.Query; - -/** - * Tests hex dumping of packets. - * - * @author Steinar Knutsen - */ -public class PacketQueryTracerTestCase { - - FS4Channel channel; - BasicPacket packet; - PacketListener tracer; - - static class MockChannel extends FS4Channel { - - @Override - public void setQuery(Query query) { - super.setQuery(query); - } - - @Override - public Query getQuery() { - return super.getQuery(); - } - - @Override - public Integer getChannelId() { - return 1; - } - - @Override - public void close() { - } - - @Override - public boolean sendPacket(BasicPacket packet) { - return true; - } - - @Override - public BasicPacket[] receivePackets(long timeout, int packetCount) { - return null; - } - - @Override - public BasicPacket nextPacket(long timeout) { - return null; - } - - @Override - protected void addPacket(BasicPacket packet) { - } - - @Override - public boolean isValid() { - return true; - } - - @Override - public String toString() { - return "MockChannel"; - } - } - - @Before - public void setUp() throws Exception { - channel = new MockChannel(); - channel.setQuery(new Query("/?query=a&tracelevel=11")); - packet = new PingPacket(); - tracer = new PacketQueryTracer(); - } - - @After - public void tearDown() throws Exception { - } - - @Test - public final void testPacketSent() throws IOException { - byte[] simulatedPacket = new byte[] { 1, 2, 3 }; - tracer.packetReceived(channel, packet, ByteBuffer.wrap(simulatedPacket)); - StringWriter w = new StringWriter(); - channel.getQuery().getContext(false).render(w); - assertTrue(w.getBuffer().toString().indexOf("PingPacket: 010203") != -1); - } - - @Test - public final void testPacketReceived() throws IOException { - byte[] simulatedPacket = new byte[] { 1, 2, 3 }; - tracer.packetReceived(channel, packet, ByteBuffer.wrap(simulatedPacket)); - StringWriter w = new StringWriter(); - channel.getQuery().getContext(false).render(w); - assertTrue(w.getBuffer().toString().indexOf("PingPacket: 010203") != -1); - } - -} diff --git a/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java b/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java deleted file mode 100644 index 8696ca08d2b..00000000000 --- a/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.fs4.mplex; - -import com.yahoo.container.QrConfig; -import com.yahoo.container.search.Fs4Config; -import com.yahoo.fs4.BasicPacket; -import com.yahoo.fs4.ChannelTimeoutException; -import com.yahoo.fs4.PingPacket; -import com.yahoo.fs4.QueryPacket; -import com.yahoo.fs4.mplex.Backend.BackendStatistics; -import com.yahoo.prelude.fastsearch.FS4ResourcePool; -import com.yahoo.search.Query; -import org.junit.Test; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.util.logging.Logger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Test networking code for talking to dispatch. - * - * @author Steinar Knutsen - */ -public class BackendTestCase { - private static final long TIMEOUT = 30000; - - public static class MockDispatch implements Runnable { - - public final ServerSocket socket; - public volatile Socket connection; - volatile int channelId; - - public byte[] packetData = new byte[] {0,0,0,104, - 0,0,0,217-256, - 0,0,0,1, - 0,0,0,0, - 0,0,0,2, - 0,0,0,0,0,0,0,5, - 0x40,0x39,0,0,0,0,0,0, - 0,0,0,111, - 0,0,0,97, - 0,0,0,3, 1,1,1,1,1,1,1,1,1,1,1,1, 0x40,0x37,0,0,0,0,0,23, 0,0,0,7, 0,0,0,36, - 0,0,0,4, 2,2,2,2,2,2,2,2,2,2,2,2, 0x40,0x35,0,0,0,0,0,21, 0,0,0,8, 0,0,0,37}; - - MockDispatch(ServerSocket socket) { - this.socket = socket; - } - - @Override - public void run() { - try { - connection = socket.accept(); - } catch (IOException e) { - e.printStackTrace(); - return; - } - requestRespond(); - } - - void requestRespond() { - byte[] length = new byte[4]; - try { - connection.getInputStream().read(length); - } catch (IOException e) { - e.printStackTrace(); - return; - } - int actual = ByteBuffer.wrap(length).getInt(); - - int read = 0; - int i = 0; - while (read != -1 && i < actual) { - try { - read = connection.getInputStream().read(); - ++i; - } catch (IOException e) { - e.printStackTrace(); - return; - } - } - ByteBuffer reply = ByteBuffer.wrap(packetData); - if (channelId != -1) { - reply.putInt(8, channelId); - } - try { - connection.getOutputStream().write(packetData); - } catch (IOException e) { - e.printStackTrace(); - } - } - public void setNoChannel() { - channelId = -1; - } - - } - - public static class MockServer { - public InetSocketAddress host; - public Thread worker; - public MockDispatch dispatch; - - public MockServer() throws IOException { - ServerSocket socket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); - host = (InetSocketAddress) socket.getLocalSocketAddress(); - dispatch = new MockDispatch(socket); - worker = new Thread(dispatch); - worker.start(); - } - - } - - private Backend backend; - private MockServer server; - private Logger logger; - private boolean initUseParent; - private FS4ResourcePool listeners; - - public static final byte[] PONG = new byte[] { 0, 0, 0, 32, 0, 0, 0, 221 - 256, - 0,0,0,1, 0, 0, 0, 42, 0, 0, 0, 127, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 1, 0, - 0, 0, 1 }; - - public void setUp() throws Exception { - logger = Logger.getLogger(Backend.class.getName()); - initUseParent = logger.getUseParentHandlers(); - logger.setUseParentHandlers(false); - listeners = new FS4ResourcePool(new Fs4Config(new Fs4Config.Builder()), new QrConfig(new QrConfig.Builder())); - - server = new MockServer(); - backend = listeners.getBackend(server.host.getHostString(), server.host.getPort()); - } - - public void tearDown() throws Exception { - listeners.deconstruct(); - server.dispatch.socket.close(); - if (server.dispatch.connection !=null) server.dispatch.connection.close(); - if (server.worker!=null) server.worker.join(); - if (logger !=null) logger.setUseParentHandlers(initUseParent); - } - - private BasicPacket [] read(FS4Channel channel) throws InvalidChannelException { - try { - return channel.receivePackets(TIMEOUT, 1); - } catch (ChannelTimeoutException e) { - fail("Could not get packets from simulated backend."); - } - return new BasicPacket[1]; - } - - @Test - public void testAll() throws Exception { - setUp(); - doTestBackend(); - tearDown(); - - setUp(); - doTestPinging(); - tearDown(); - - setUp(); - doRequireStatistics(); - tearDown(); - } - - private void doTestBackend() throws IOException, InvalidChannelException { - FS4Channel channel = backend.openChannel(); - Query q = new Query("/?query=a"); - int channelId = channel.getChannelId(); - server.dispatch.channelId = channelId; - - assertTrue(backend.sendPacket(QueryPacket.create("container.0", q), channelId)); - BasicPacket[] b = read(channel); - assertEquals(1, b.length); - assertEquals(217, b[0].getCode()); - channel.close(); - } - - private void doTestPinging() throws IOException, InvalidChannelException { - FS4Channel channel = backend.openPingChannel(); - server.dispatch.setNoChannel(); - server.dispatch.packetData = PONG; - - assertTrue(channel.sendPacket(new PingPacket())); - BasicPacket[] b = read(channel); - assertEquals(1, b.length); - assertEquals(221, b[0].getCode()); - channel.close(); - } - - private void doRequireStatistics() throws IOException, InvalidChannelException { - FS4Channel channel = backend.openPingChannel(); - server.dispatch.channelId = -1; - server.dispatch.packetData = PONG; - - assertTrue(channel.sendPacket(new PingPacket())); - read(channel); - BackendStatistics stats = backend.getStatistics(); - assertEquals(1, stats.totalConnections()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java deleted file mode 100644 index 527a2736fee..00000000000 --- a/container-search/src/test/java/com/yahoo/fs4/test/GetDocSumsPacketTestCase.java +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.fs4.test; - -import com.yahoo.fs4.BufferTooSmallException; -import com.yahoo.fs4.GetDocSumsPacket; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.query.SessionId; -import com.yahoo.search.result.Hit; -import org.junit.Test; - -import java.nio.ByteBuffer; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -/** - * Tests the GetDocsumsPacket - * - * @author Bjorn Borud - */ -public class GetDocSumsPacketTestCase { - - private static final byte IGNORE = 69; - - @Test - public void testDefaultDocsumClass() { - Query query = new Query("/?query=chain"); - assertNull(query.getPresentation().getSummary()); - } - - @Test - public void testEncodingWithQuery() throws BufferTooSmallException { - Hit hit = new FastHit(); - assertPacket(true, hit, new byte[] {0, 0, 0, 53, 0, 0, 0, -37, 0, 0, 8, 21, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, - IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x03, 0, 0, 0, 7, - 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 6, 4, 0, 3, 102, 111, 111}); - } - - @Test - public void testEncodingWithoutQuery() throws BufferTooSmallException { - Hit hit = new FastHit(); - assertPacket(false, hit, new byte[] { 0, 0, 0, 39, 0, 0, 0, -37, 0, 0, 8, 17, 0, 0, 0, 0, IGNORE, IGNORE, IGNORE, - IGNORE, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 0x03, 0, 0, 0, 7, 100, 101, 102, 97, 117, 108, 116 - }); - } - - @Test - public void requireThatSessionIdIsEncodedAsPropertyWhenUsingSearchSession() throws BufferTooSmallException { - Result result = new Result(new Query("?query=foo&groupingSessionCache=false")); - SessionId sessionId = result.getQuery().getSessionId("node-0"); - result.getQuery().getRanking().setQueryCache(true); - FastHit hit = new FastHit(); - result.hits().add(hit); - ByteBuffer answer = ByteBuffer.allocate(1024); - //assertEquals(0, sessionId.asUtf8String().getByteLength()); - answer.put(new byte[] { 0, 0, 0, (byte)(103+sessionId.asUtf8String().getByteLength()), 0, 0, 0, -37, 0, 0, 24, 17, 0, 0, 0, 0, - // query timeout - IGNORE, IGNORE, IGNORE, IGNORE, - // "default" - rank profile - 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 0x03, - // "default" - summaryclass - 0, 0, 0, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', - // 2 property entries - 0, 0, 0, 2, - // rank: sessionId => qrserver.0.XXXXXXXXXXXXX.0 - 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 1, 0, 0, 0, 9, 's', 'e', 's', 's', 'i', 'o', 'n', 'I', 'd'}); - answer.putInt(sessionId.asUtf8String().getByteLength()); - answer.put(sessionId.asUtf8String().getBytes()); - answer.put(new byte [] { - // caches: features => true - 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', - 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e'}); - byte [] expected = new byte [answer.position()]; - answer.flip(); - answer.get(expected); - assertPacket(false, result, expected); - } - - private static void assertPacket(boolean sendQuery, Hit hit, byte[] expected) throws BufferTooSmallException { - Result result = new Result(new Query("?query=foo")); - result.hits().add(hit); - assertPacket(sendQuery, result, expected); - } - - private static void assertPacket(boolean sendQuery, Result result, byte[] expected) throws BufferTooSmallException { - GetDocSumsPacket packet = GetDocSumsPacket.create(result, "default", sendQuery); - ByteBuffer buf = ByteBuffer.allocate(1024); - packet.encode(buf); - buf.flip(); - - byte[] actual = new byte[buf.remaining()]; - buf.get(actual); - // assertEquals(Arrays.toString(expected), Arrays.toString(actual)); - - assertEquals("Equal length", expected.length, actual.length); - for (int i = 0; i < expected.length; ++i) { - if (expected[i] == IGNORE) { - actual[i] = IGNORE; - } - } - - assertArrayEquals(expected, actual); - } -} diff --git a/container-search/src/test/java/com/yahoo/fs4/test/HexByteIteratorTestCase.java b/container-search/src/test/java/com/yahoo/fs4/test/HexByteIteratorTestCase.java deleted file mode 100644 index 022feb09b2e..00000000000 --- a/container-search/src/test/java/com/yahoo/fs4/test/HexByteIteratorTestCase.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.fs4.test; - -import java.nio.ByteBuffer; - -import com.yahoo.fs4.HexByteIterator; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Test of HexByteIterator - * - * @author Tony Vaagenes - */ -public class HexByteIteratorTestCase { - - @Test - public void testHexByteIterator() { - int[] numbers = { 0x00, 0x01, 0xDE, 0xAD, 0xBE, 0xEF, 0xFF }; - - HexByteIterator i = new HexByteIterator( - ByteBuffer.wrap(toBytes(numbers))); - - assertEquals("00", i.next()); - assertEquals("01", i.next()); - assertEquals("DE", i.next()); - assertEquals("AD", i.next()); - assertEquals("BE", i.next()); - assertEquals("EF", i.next()); - assertEquals("FF", i.next()); - assertTrue(!i.hasNext()); - } - - private byte[] toBytes(int[] ints) { - byte[] bytes = new byte[ints.length]; - for (int i=0; i entries) { - RankProperties properties = createRankPropertiesWithTensors(entries); - assertEquals(entries.size(), properties.asMap().size()); - - Map decodedProperties = decode(type, encode(properties)); - assertEquals(entries.size(), properties.asMap().size()); - assertEquals(entries.size(), decodedProperties.size()); - for (Entry entry : entries) { - assertEquals(entry.tensor, decodedProperties.get(entry.normalizedKey)); - } - } - - private static void assertTensorEncodingAndDecoding(TensorType type, String key, String normalizedKey, Tensor tensor) { - assertTensorEncodingAndDecoding(type, Arrays.asList(new Entry(key, normalizedKey, tensor))); - } - - private static RankProperties createRankPropertiesWithTensors(List entries) { - RankFeatures features = new RankFeatures(); - for (Entry entry : entries) { - features.put(entry.key, entry.tensor); - } - RankProperties properties = new RankProperties(); - features.prepare(properties); - return properties; - } - - private static byte[] encode(RankProperties properties) { - ByteBuffer buffer = ByteBuffer.allocate(512); - properties.encode(buffer, true); - byte[] result = new byte[buffer.position()]; - buffer.rewind(); - buffer.get(result); - return result; - } - - private static Map decode(TensorType type, byte[] encodedProperties) { - GrowableByteBuffer buffer = GrowableByteBuffer.wrap(encodedProperties); - byte[] mapNameBytes = new byte[buffer.getInt()]; - buffer.get(mapNameBytes); - int numEntries = buffer.getInt(); - Map result = new HashMap<>(); - for (int i = 0; i < numEntries; ++i) { - byte[] keyBytes = new byte[buffer.getInt()]; - buffer.get(keyBytes); - String key = Utf8.toString(keyBytes); - byte[] value = new byte[buffer.getInt()]; - buffer.get(value); - if (key.contains(".type")) { - result.put(key, Utf8.toString(value)); - } else { - result.put(key, TypedBinaryFormat.decode(Optional.of(type), GrowableByteBuffer.wrap(value))); - } - } - return result; - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 86553a86add..9ea7276583b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -7,8 +7,6 @@ import com.yahoo.container.QrConfig; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.protect.Error; -import com.yahoo.container.search.Fs4Config; -import com.yahoo.fs4.QueryPacket; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; @@ -48,12 +46,12 @@ import static org.junit.Assert.assertTrue; * @author bratseth */ public class ClusterSearcherTestCase { + private static final double DELTA = 0.0000000000000001; @Test public void testNoBackends() { ClusterSearcher cluster = new ClusterSearcher(new LinkedHashSet<>(Arrays.asList("dummy"))); try { - cluster.getMonitor().getConfiguration().setRequestTimeout(100); Execution execution = new Execution(cluster, Execution.Context.createContextStub()); Query query = new Query("query=hello"); query.setHits(10); @@ -146,7 +144,7 @@ public class ClusterSearcherTestCase { private final String type3 = "type3"; private final Map> results = new LinkedHashMap<>(); private final boolean expectAttributePrefetch; - public static final String ATTRIBUTE_PREFETCH = "attributeprefetch"; + static final String ATTRIBUTE_PREFETCH = "attributeprefetch"; private String getId(String type, int i) { return "id:ns:" + type + "::" + i; @@ -196,7 +194,7 @@ public class ClusterSearcherTestCase { createHit(getId(type3, 2), 5))); } - public MyMockSearcher(boolean expectAttributePrefetch) { + MyMockSearcher(boolean expectAttributePrefetch) { this.expectAttributePrefetch = expectAttributePrefetch; init(); } @@ -263,8 +261,7 @@ public class ClusterSearcherTestCase { } private Execution createExecution(List docTypesList, boolean expectAttributePrefetch) { - Set documentTypes = new LinkedHashSet<>(); - documentTypes.addAll(docTypesList); + Set documentTypes = new LinkedHashSet<>(docTypesList); ClusterSearcher cluster = new ClusterSearcher(documentTypes); try { cluster.addBackendSearcher(new MyMockSearcher( @@ -277,6 +274,7 @@ public class ClusterSearcherTestCase { } } + @Test public void testThatSingleDocumentTypeCanBeSearched() { { // Explicit 1 type in restrict set Execution execution = createExecution(); @@ -285,9 +283,9 @@ public class ClusterSearcherTestCase { assertEquals(3, result.getTotalHitCount()); List hits = result.hits().asList(); assertEquals(3, hits.size()); - assertEquals(9.0, hits.get(0).getRelevance().getScore()); - assertEquals(6.0, hits.get(1).getRelevance().getScore()); - assertEquals(3.0, hits.get(2).getRelevance().getScore()); + assertEquals(9.0, hits.get(0).getRelevance().getScore(), DELTA); + assertEquals(6.0, hits.get(1).getRelevance().getScore(), DELTA); + assertEquals(3.0, hits.get(2).getRelevance().getScore(), DELTA); } { // Only 1 registered type in cluster searcher, empty restrict set // NB ! Empty restrict sets does not exist below the cluster searcher. @@ -299,10 +297,11 @@ public class ClusterSearcherTestCase { assertEquals(3, result.getTotalHitCount()); List hits = result.hits().asList(); assertEquals(3, hits.size()); - assertEquals(9.0, hits.get(0).getRelevance().getScore()); + assertEquals(9.0, hits.get(0).getRelevance().getScore(), DELTA); } } + @Test public void testThatSubsetOfDocumentTypesCanBeSearched() { Execution execution = createExecution(); Query query = new Query("?query=hello&restrict=type1,type3"); @@ -311,14 +310,15 @@ public class ClusterSearcherTestCase { assertEquals(6, result.getTotalHitCount()); List hits = result.hits().asList(); assertEquals(6, hits.size()); - assertEquals(11.0, hits.get(0).getRelevance().getScore()); - assertEquals(9.0, hits.get(1).getRelevance().getScore()); - assertEquals(8.0, hits.get(2).getRelevance().getScore()); - assertEquals(6.0, hits.get(3).getRelevance().getScore()); - assertEquals(5.0, hits.get(4).getRelevance().getScore()); - assertEquals(3.0, hits.get(5).getRelevance().getScore()); + assertEquals(11.0, hits.get(0).getRelevance().getScore(), DELTA); + assertEquals(9.0, hits.get(1).getRelevance().getScore(), DELTA); + assertEquals(8.0, hits.get(2).getRelevance().getScore(), DELTA); + assertEquals(6.0, hits.get(3).getRelevance().getScore(), DELTA); + assertEquals(5.0, hits.get(4).getRelevance().getScore(), DELTA); + assertEquals(3.0, hits.get(5).getRelevance().getScore(), DELTA); } + @Test public void testThatMultipleDocumentTypesCanBeSearchedAndFilled() { Execution execution = createExecution(); Query query = new Query("?query=hello"); @@ -327,15 +327,15 @@ public class ClusterSearcherTestCase { assertEquals(9, result.getTotalHitCount()); List hits = result.hits().asList(); assertEquals(9, hits.size()); - assertEquals(11.0, hits.get(0).getRelevance().getScore()); - assertEquals(10.0, hits.get(1).getRelevance().getScore()); - assertEquals(9.0, hits.get(2).getRelevance().getScore()); - assertEquals(8.0, hits.get(3).getRelevance().getScore()); - assertEquals(7.0, hits.get(4).getRelevance().getScore()); - assertEquals(6.0, hits.get(5).getRelevance().getScore()); - assertEquals(5.0, hits.get(6).getRelevance().getScore()); - assertEquals(4.0, hits.get(7).getRelevance().getScore()); - assertEquals(3.0, hits.get(8).getRelevance().getScore()); + assertEquals(11.0, hits.get(0).getRelevance().getScore(), DELTA); + assertEquals(10.0, hits.get(1).getRelevance().getScore(), DELTA); + assertEquals(9.0, hits.get(2).getRelevance().getScore(), DELTA); + assertEquals(8.0, hits.get(3).getRelevance().getScore(), DELTA); + assertEquals(7.0, hits.get(4).getRelevance().getScore(), DELTA); + assertEquals(6.0, hits.get(5).getRelevance().getScore(), DELTA); + assertEquals(5.0, hits.get(6).getRelevance().getScore(), DELTA); + assertEquals(4.0, hits.get(7).getRelevance().getScore(), DELTA); + assertEquals(3.0, hits.get(8).getRelevance().getScore(), DELTA); for (int i = 0; i < 9; ++i) { assertNull(hits.get(i).getField("score")); } @@ -390,7 +390,7 @@ public class ClusterSearcherTestCase { assertResult(9, Arrays.asList(5.0, 4.0), getResult(6, 2, ex)); assertResult(9, Arrays.asList(4.0, 3.0), getResult(7, 2, ex)); assertResult(9, Arrays.asList(3.0), getResult(8, 2, ex)); - assertResult(9, new ArrayList(), getResult(9, 2, ex)); + assertResult(9, new ArrayList<>(), getResult(9, 2, ex)); assertResult(9, Arrays.asList(11.0, 10.0, 9.0, 8.0, 7.0), getResult(0, 5, ex)); assertResult(9, Arrays.asList(6.0, 5.0, 4.0, 3.0), getResult(5, 5, ex)); @@ -425,11 +425,7 @@ public class ClusterSearcherTestCase { final String yahoo = "www.yahoo.com"; try { - if (null != InetAddress.getByName(yahoo)) { - canFindYahoo = true; - } else { - canFindYahoo = false; - } + canFindYahoo = (null != InetAddress.getByName(yahoo)); } catch (Exception e) { canFindYahoo = false; } @@ -542,12 +538,11 @@ public class ClusterSearcherTestCase { qrSearchersConfig.build(), clusterConfig.build(), documentDbConfig.build(), - new QrMonitorConfig.Builder().build(), new DispatchConfig.Builder().build(), createClusterInfoConfig(), Statistics.nullImplementation, new MockMetric(), - new FS4ResourcePool(new Fs4Config.Builder().build(), new QrConfig.Builder().build()), + new FS4ResourcePool(new QrConfig.Builder().build()), new VipStatus()); } @@ -590,7 +585,7 @@ public class ClusterSearcherTestCase { @Test public void testThatQueryTimeoutIsCappedWithSpecifiedMax() { - QueryTimeoutFixture f = new QueryTimeoutFixture(Double.valueOf(70), null); + QueryTimeoutFixture f = new QueryTimeoutFixture(70.0, null); f.query.setTimeout(70001); f.search(); assertEquals(70000, f.query.getTimeout()); @@ -616,7 +611,7 @@ public class ClusterSearcherTestCase { @Test public void testThatQueryCacheIsDisabledIfTimeoutIsLargerThanConfiguredMax() { - QueryTimeoutFixture f = new QueryTimeoutFixture(null, Double.valueOf(5)); + QueryTimeoutFixture f = new QueryTimeoutFixture(null, 5.0); f.query.setTimeout(5001); f.query.getRanking().setQueryCache(true); f.search(); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/FS4SearchInvokerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/FS4SearchInvokerTestCase.java deleted file mode 100644 index b9119528490..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/FS4SearchInvokerTestCase.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.prelude.fastsearch; - -import com.yahoo.fs4.BasicPacket; -import com.yahoo.fs4.mplex.FS4Channel; -import com.yahoo.fs4.mplex.InvalidChannelException; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.dispatch.InterleavedSearchInvoker; -import com.yahoo.search.dispatch.MockSearchCluster; -import com.yahoo.search.dispatch.ResponseMonitor; -import com.yahoo.search.searchchain.Execution; -import org.hamcrest.Matchers; -import org.junit.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; - -import static org.junit.Assert.assertThat; - -public class FS4SearchInvokerTestCase { - @SuppressWarnings("resource") - @Test - public void testThatConnectionErrorsAreReportedImmediately() throws IOException { - var query = new Query("?"); - query.setTimeout(1000); - - var searcher = mockSearcher(); - var cluster = new MockSearchCluster("?", 1, 1); - var fs4invoker = new FS4SearchInvoker(searcher, query, mockFailingChannel(), Optional.empty()); - var interleave = new InterleavedSearchInvoker(Collections.singleton(fs4invoker), cluster, null); - - long start = System.currentTimeMillis(); - interleave.search(query, null); - long elapsed = System.currentTimeMillis() - start; - - assertThat("Connection error should fail fast", elapsed, Matchers.lessThan(500L)); - } - - private static VespaBackEndSearcher mockSearcher() { - return new VespaBackEndSearcher() { - @Override - protected Result doSearch2(Query query, Execution execution) { - return null; - } - - @Override - protected void doPartialFill(Result result, String summaryClass) {} - }; - } - - private static FS4Channel mockFailingChannel() { - return new FS4Channel() { - @Override - public boolean sendPacket(BasicPacket packet) throws InvalidChannelException, IOException { - // pretend there's a connection error - return false; - } - - @Override - public void setQuery(Query q) {} - - @Override - public void setResponseMonitor(ResponseMonitor monitor) {} - - @Override - public void close() {} - }; - } -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java deleted file mode 100644 index b0662a93f62..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/DirectSearchTestCase.java +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test; - -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.search.Result; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * Tests that FastSearcher will bypass dispatch when the conditions are right - * - * @author bratseth - */ -public class DirectSearchTestCase { - - @Test - public void testDirectSearchEnabled() { - FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0"); - tester.search("?query=test&dispatch.direct=true"); - assertEquals("The FastSearcher has used the local search node connection", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - } - - @Test - public void testDirectSearchDisabled() { - FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0"); - tester.search("?query=test&dispatch.direct=false&dispatch.internal=false"); - assertEquals(0, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - } - - @Test - public void testDirectSearchEnabledByDefault() { - FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0"); - tester.search("?query=test"); - assertEquals("The FastSearcher has used the local search node connection", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - } - - @Test - public void testNoDirectSearchWhenMoreSearchNodesThanContainers() { - FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0", "otherhost:9999:1"); - tester.search("?query=test&dispatch.direct=true&dispatch.internal=false"); - assertEquals(0, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - } - - @Test - public void testDirectSearchWhenMultipleGroupsAndEnoughContainers() { - FastSearcherTester tester = new FastSearcherTester(2, FastSearcherTester.selfHostname + ":9999:0", "otherhost:9999:1"); - tester.search("?query=test&dispatch.direct=true"); - assertEquals(1, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - } - - @Test - public void testDirectSearchSummaryFetchGoToLocalNode() { - FastSearcherTester tester = new FastSearcherTester(2, "otherhost:9999:1", FastSearcherTester.selfHostname + ":9999:0"); - int localDistributionKey = tester.dispatcher().searchCluster().nodesByHost().get(FastSearcherTester.selfHostname).asList().get(0).key(); - assertEquals(1, localDistributionKey); - Result result = tester.search("?query=test&dispatch.direct=true"); - assertEquals(1, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - FastHit hit = (FastHit)result.hits().get(0); - assertEquals(localDistributionKey, hit.getDistributionKey()); - } - - @Test - public void testNoDirectSearchWhenMultipleNodesPerGroup() { - FastSearcherTester tester = new FastSearcherTester(2, FastSearcherTester.selfHostname + ":9999:0", "otherhost:9999:0"); - tester.search("?query=test&dispatch.direct=true&dispatch.internal=false"); - assertEquals(0, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - } - - @Test - public void testNoDirectSearchWhenLocalNodeIsDown() { - FastSearcherTester tester = new FastSearcherTester(2, FastSearcherTester.selfHostname + ":9999:0", "otherhost:9999:1"); - assertTrue(tester.vipStatus().isInRotation()); - tester.setResponding(FastSearcherTester.selfHostname, false); - assertFalse(tester.vipStatus().isInRotation()); - assertEquals("1 ping request, 0 search requests", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.search("?query=test&dispatch.direct=true&nocache"); - assertEquals("1 ping request, 0 search requests", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.setResponding(FastSearcherTester.selfHostname, true); - assertTrue(tester.vipStatus().isInRotation()); - assertEquals("2 ping requests, 0 search request", 2, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.search("?query=test&dispatch.direct=true&nocache"); - assertEquals("2 ping requests, 1 search request", 3, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - } - - @Test - public void testNoDirectDispatchWhenInsufficientCoverage() { - FastSearcherTester tester = new FastSearcherTester(3, - FastSearcherTester.selfHostname + ":9999:0", - "host1:9999:1", - "host2:9999:2"); - double k = 38.78955; // multiply all document counts by some number > 1 to test that we compute % correctly - - tester.setActiveDocuments(FastSearcherTester.selfHostname, (long) (96 * k)); - tester.setActiveDocuments("host1", (long) (100 * k)); - tester.setActiveDocuments("host2", (long) (100 * k)); - assertEquals("1 ping request, 0 search requests", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.search("?query=test&dispatch.direct=true&nocache"); - assertEquals("Still 1 ping request, 0 search requests because the default coverage is 97%, and we only have 96% locally", - 1, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.waitForInRotationIs(false); - - tester.setActiveDocuments(FastSearcherTester.selfHostname, (long) (99 * k)); - assertEquals("2 ping request, 0 search requests", 2, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.search("?query=test&dispatch.direct=true&nocache"); - assertEquals("2 ping request, 1 search requests because we now have 99% locally", - 3, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.waitForInRotationIs(true); - - tester.setActiveDocuments("host1", (long) (104 * k)); - assertEquals("2 ping request, 1 search requests", 3, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.search("?query=test&dispatch.direct=true&nocache"); - assertEquals("2 ping request, 2 search requests because 99/((104+100)/2) > 0.97", - 4, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.waitForInRotationIs(true); - - tester.setActiveDocuments("host2", (long) (102 * k)); - assertEquals("2 ping request, 2 search requests", 4, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.search("?query=test&dispatch.direct=true&nocache"); - assertEquals("Still 2 ping request, 2 search requests because 99/((104+102)/2) < 0.97", - 4, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.waitForInRotationIs(false); - } - - @Test - public void testCoverageWithSingleGroup() { - FastSearcherTester tester = new FastSearcherTester(1, FastSearcherTester.selfHostname + ":9999:0"); - - tester.setActiveDocuments(FastSearcherTester.selfHostname, 100); - assertEquals("1 ping request, 0 search requests", 1, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - tester.search("?query=test&dispatch.direct=true&nocache"); - assertEquals("1 ping request, 1 search requests", 2, tester.requestCount(FastSearcherTester.selfHostname, 9999)); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index c6e87170f07..eb4d65693bb 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -3,64 +3,34 @@ package com.yahoo.prelude.fastsearch.test; import com.google.common.collect.ImmutableList; import com.yahoo.component.chain.Chain; -import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.container.QrConfig; -import com.yahoo.container.handler.VipStatus; import com.yahoo.container.protect.Error; -import com.yahoo.container.search.Fs4Config; -import com.yahoo.document.GlobalId; -import com.yahoo.fs4.BasicPacket; -import com.yahoo.fs4.Packet; -import com.yahoo.fs4.mplex.Backend; -import com.yahoo.fs4.mplex.BackendTestCase; -import com.yahoo.fs4.test.QueryTestCase; import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; import com.yahoo.prelude.fastsearch.ClusterParams; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FS4ResourcePool; -import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.FastSearcher; import com.yahoo.prelude.fastsearch.SummaryParameters; -import com.yahoo.prelude.fastsearch.test.fs4mock.MockBackend; -import com.yahoo.prelude.fastsearch.test.fs4mock.MockFS4ResourcePool; -import com.yahoo.prelude.fastsearch.test.fs4mock.MockFSChannel; -import com.yahoo.processing.execution.Execution.Trace; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; -import com.yahoo.search.dispatch.rpc.MockRpcResourcePoolBuilder; import com.yahoo.search.dispatch.searchcluster.Node; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.grouping.request.AllOperation; import com.yahoo.search.grouping.request.EachOperation; import com.yahoo.search.grouping.request.GroupingOperation; -import com.yahoo.search.query.SessionId; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; -import com.yahoo.yolean.trace.TraceNode; -import com.yahoo.yolean.trace.TraceVisitor; import org.junit.Test; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; -import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; + /** * Tests the Fast searcher @@ -70,30 +40,12 @@ import static org.junit.Assert.assertTrue; public class FastSearcherTestCase { private final static DocumentdbInfoConfig documentdbInfoConfig = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder()); - private MockBackend mockBackend; - @Test - public void testNoNormalizing() { - Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); - FastSearcher fastSearcher = new FastSearcher(new MockBackend(), - new FS4ResourcePool("container.0", 1), - MockDispatcher.create(Collections.emptyList()), - new SummaryParameters(null), - new ClusterParams("testhittype"), - documentdbInfoConfig); - - MockFSChannel.setEmptyDocsums(false); - - Result result = doSearch(fastSearcher, new Query("?query=ignored"), 0, 10); - - assertTrue(result.hits().get(0).getRelevance().getScore() > 1000); - } @Test public void testNullQuery() { Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); - FastSearcher fastSearcher = new FastSearcher(new MockBackend(), - new FS4ResourcePool("container.0", 1), + FastSearcher fastSearcher = new FastSearcher("container.0", MockDispatcher.create(Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -109,152 +61,6 @@ public class FastSearcherTestCase { assertEquals(Error.NULL_QUERY.code, message.getCode()); } - @Test - public void testDispatchDotSummaries() { - Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); - DocumentdbInfoConfig documentdbConfigWithOneDb = - new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder() - .name("testDb") - .summaryclass(new DocumentdbInfoConfig.Documentdb.Summaryclass.Builder().name("simple").id(7)) - .rankprofile(new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder() - .name("simpler").hasRankFeatures(false).hasSummaryFeatures(false)))); - - List nodes = new ArrayList<>(); - nodes.add(new Node(0, "host1", 5000, 0)); - nodes.add(new Node(1, "host2", 5000, 0)); - - var mockFs4ResourcePool = new MockFS4ResourcePool(); - var mockRpcResourcePool = new MockRpcResourcePoolBuilder().connection(0).connection(1).build(); - - FastSearcher fastSearcher = new FastSearcher(new MockBackend(), - mockFs4ResourcePool, - MockDispatcher.create(nodes, mockFs4ResourcePool, mockRpcResourcePool, 1, new VipStatus()), - new SummaryParameters(null), - new ClusterParams("testhittype"), - documentdbConfigWithOneDb); - - { // No direct.summaries - String query = "?query=sddocname:a&summary=simple&timeout=20s"; - Result result = doSearch(fastSearcher, new Query(query), 0, 10); - doFill(fastSearcher, result); - ErrorMessage error = result.hits().getError(); - assertNull("Since we don't route to the dispatcher we hit the mock backend, so no error", error); - } - - { // direct.summaries due to query cache - String query = "?query=sddocname:a&ranking.queryCache&timeout=20s"; - Result result = doSearch(fastSearcher, new Query(query), 0, 10); - doFill(fastSearcher, result); - ErrorMessage error = result.hits().getError(); - assertEquals("Since we don't actually run summary backends we get this error when the Dispatcher is used", - "getDocsums(..) attempted for node X", error.getDetailedMessage().replaceAll("\\d", "X")); - } - - { // direct.summaries due to no summary features - String query = "?query=sddocname:a&dispatch.summaries&summary=simple&ranking=simpler&timeout=20s"; - Result result = doSearch(fastSearcher, new Query(query), 0, 10); - doFill(fastSearcher, result); - ErrorMessage error = result.hits().getError(); - assertEquals("Since we don't actually run summary backends we get this error when the Dispatcher is used", - "getDocsums(..) attempted for node X", error.getDetailedMessage().replaceAll("\\d", "X")); - } - } - - @Test - public void testQueryWithRestrict() { - mockBackend = new MockBackend(); - DocumentdbInfoConfig documentdbConfigWithOneDb = - new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("testDb"))); - FastSearcher fastSearcher = new FastSearcher(mockBackend, - new FS4ResourcePool("container.0", 1), - MockDispatcher.create(Collections.emptyList()), - new SummaryParameters(null), - new ClusterParams("testhittype"), - documentdbConfigWithOneDb); - - Query query = new Query("?query=foo&model.restrict=testDb&groupingSessionCache=false"); - query.prepare(); - doSearch(fastSearcher, query, 0, 10); - - Packet receivedPacket = mockBackend.getChannel().getLastQueryPacket(); - byte[] encoded = QueryTestCase.packetToBytes(receivedPacket); - byte[] correct = new byte[] { - 0, 0, 0, 100, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, 6, 0, 10, - QueryTestCase.ignored, QueryTestCase.ignored, QueryTestCase.ignored, QueryTestCase.ignored, // time left - 0, 0, 0x40, 0x03, 7, 100, 101, 102, 97, 117, 108, 116, 0, 0, 0, 1, 0, 0, 0, 5, 109, 97, 116, 99, 104, 0, 0, 0, 1, 0, 0, 0, 24, 100, 111, 99, 117, 109, 101, 110, 116, 100, 98, 46, 115, 101, 97, 114, 99, 104, 100, 111, 99, 116, 121, 112, 101, 0, 0, 0, 6, 116, 101, 115, 116, 68, 98, 0, 0, 0, 1, 0, 0, 0, 7, 68, 1, 0, 3, 102, 111, 111 - }; - QueryTestCase.assertEqualArrays(correct, encoded); - } - - @Test - public void testSearch() { - FastSearcher fastSearcher = createFastSearcher(); - - Result result = doSearch(fastSearcher, new Query("?query=ignored"), 0, 10); - - Execution execution = new Execution(chainedAsSearchChain(fastSearcher), Execution.Context.createContextStub()); - assertEquals(2, result.getHitCount()); - execution.fill(result); - assertCorrectHit1((FastHit)result.hits().get(0)); - assertCorrectTypes1((FastHit)result.hits().get(0)); - for (int idx = 0; idx < result.getHitCount(); idx++) { - assertTrue(!result.hits().get(idx).isCached()); - } - - // Repeat the request a couple of times, to verify whether the packet cache works - result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 10); - assertEquals(2, result.getHitCount()); - execution.fill(result); - assertCorrectHit1((FastHit) result.hits().get(0)); - for (int i = 0; i < result.getHitCount(); i++) { - assertFalse(result.hits().get(i) + " should never be cached", - result.hits().get(i).isCached()); - } - - result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 10); - assertEquals(2, result.getHitCount()); - execution.fill(result); - assertCorrectHit1((FastHit) result.hits().get(0)); - assertTrue("All hits are not cached", !result.isCached()); - for (int i = 0; i < result.getHitCount(); i++) { - assertTrue(!result.hits().get(i).isCached()); - } - - // Test that partial result sets can be retrieved from the cache - result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 1); - assertEquals(1, result.getConcreteHitCount()); - execution.fill(result); - - result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 2); - assertEquals(2, result.getConcreteHitCount()); - execution.fill(result); - // No hit should be cached - assertFalse(result.hits().get(0).isCached()); - assertFalse(result.hits().get(1).isCached()); - - // Still nothing cached - result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 2); - assertEquals(2, result.getConcreteHitCount()); - execution.fill(result); - // both first and second should now be cached - assertFalse(result.hits().get(0).isCached()); - assertFalse(result.hits().get(1).isCached()); - - // Tests that the cache _hit_ is not returned if _another_ - // hit is requested - - result = doSearch(fastSearcher,new Query("?query=ignored"), 0, 1); - assertEquals(1, result.getConcreteHitCount()); - - result = doSearch(fastSearcher,new Query("?query=ignored"), 1, 1); - assertEquals(1, result.getConcreteHitCount()); - - for (int i = 0; i < result.getHitCount(); i++) { - assertFalse("Hit " + i + " should not be cached.", - result.hits().get(i).isCached()); - } - } - private Chain chainedAsSearchChain(Searcher topOfChain) { List searchers = new ArrayList<>(); searchers.add(topOfChain); @@ -272,79 +78,9 @@ public class FastSearcherTestCase { return new Execution(chainedAsSearchChain(searcher), context); } - private void doFill(Searcher searcher, Result result) { - createExecution(searcher).fill(result); - } - - @Test - public void testThatPropertiesAreReencoded() throws Exception { - FastSearcher fastSearcher = createFastSearcher(); - - Query query = new Query("?query=ignored&dispatch.summaries=false&groupingSessionCache=false"); - query.getRanking().setQueryCache(true); - Result result = doSearch(fastSearcher, query, 0, 10); - - Execution execution = new Execution(chainedAsSearchChain(fastSearcher), Execution.Context.createContextStub()); - assertEquals(2, result.getHitCount()); - execution.fill(result); - - BasicPacket receivedPacket = mockBackend.getChannel().getLastReceived(); - ByteBuffer buf = ByteBuffer.allocate(1000); - receivedPacket.encode(buf); - buf.flip(); - byte[] actual = new byte[buf.remaining()]; - buf.get(actual); - - SessionId sessionId = query.getSessionId(); - byte IGNORE = 69; - ByteBuffer answer = ByteBuffer.allocate(1024); - answer.put(new byte[] { 0, 0, 0, (byte)(141+sessionId.asUtf8String().getByteLength()), 0, 0, 0, -37, 0, 0, 16, 17, 0, 0, 0, 0, - // query timeout - IGNORE, IGNORE, IGNORE, IGNORE, - // "default" - rank profile - 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 0x03, - // 3 property entries (rank, match, caches) - 0, 0, 0, 3, - // rank: sessionId => qrserver.0.XXXXXXXXXXXXX.0 - 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 1, 0, 0, 0, 9, 's', 'e', 's', 's', 'i', 'o', 'n', 'I', 'd'}); - answer.putInt(sessionId.asUtf8String().getBytes().length); - answer.put(sessionId.asUtf8String().getBytes()); - answer.put(new byte [] { - // match: documentdb.searchdoctype => test - 0, 0, 0, 5, 'm', 'a', 't', 'c', 'h', 0, 0, 0, 1, 0, 0, 0, 24, 'd', 'o', 'c', 'u', 'm', 'e', 'n', 't', 'd', 'b', '.', 's', 'e', 'a', 'r', 'c', 'h', 'd', 'o', 'c', 't', 'y', 'p', 'e', 0, 0, 0, 4, 't', 'e', 's', 't', - // sessionId => qrserver.0.XXXXXXXXXXXXX.0 - 0, 0, 0, 6, 'c', 'a', 'c', 'h', 'e', 's', 0, 0, 0, 1, 0, 0, 0, 5, 'q', 'u', 'e', 'r', 'y', 0, 0, 0, 4, 't', 'r', 'u', 'e'}); - byte [] expected = new byte [answer.position()]; - answer.flip(); - answer.get(expected); - - for (int i = 0; i < expected.length; ++i) { - if (expected[i] == IGNORE) { - actual[i] = IGNORE; - } - } - assertArrayEquals(expected, actual); - } - - private FastSearcher createFastSearcher() { - mockBackend = new MockBackend(); - ConfigGetter getter = new ConfigGetter<>(DocumentdbInfoConfig.class); - DocumentdbInfoConfig config = getter.getConfig("file:src/test/java/com/yahoo/prelude/fastsearch/test/documentdb-info.cfg"); - - MockFSChannel.resetDocstamp(); - Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); - return new FastSearcher(mockBackend, - new FS4ResourcePool("container.0", 1), - MockDispatcher.create(Collections.emptyList()), - new SummaryParameters(null), - new ClusterParams("testhittype"), - config); - } - @Test public void testSinglePassGroupingIsForcedWithSingleNodeGroups() { - FastSearcher fastSearcher = new FastSearcher(new MockBackend(), - new FS4ResourcePool("container.0", 1), + FastSearcher fastSearcher = new FastSearcher("container.0", MockDispatcher.create(Collections.singletonList(new Node(0, "host0", 123, 0))), new SummaryParameters(null), new ClusterParams("testhittype"), @@ -368,8 +104,7 @@ public class FastSearcherTestCase { public void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { MockDispatcher dispatcher = MockDispatcher.create(ImmutableList.of(new Node(0, "host0", 123, 0), new Node(2, "host1", 123, 0))); - FastSearcher fastSearcher = new FastSearcher(new MockBackend(), - new FS4ResourcePool("container.0", 1), + FastSearcher fastSearcher = new FastSearcher("container.0", dispatcher, new SummaryParameters(null), new ClusterParams("testhittype"), @@ -401,95 +136,4 @@ public class FastSearcherTestCase { assertForceSinglePassIs(expected, child); } - @Test - public void testPing() throws IOException, InterruptedException { - Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); - BackendTestCase.MockServer server = new BackendTestCase.MockServer(); - FS4ResourcePool listeners = new FS4ResourcePool(new Fs4Config(new Fs4Config.Builder()), new QrConfig(new QrConfig.Builder())); - Backend backend = listeners.getBackend(server.host.getHostString(),server.host.getPort()); - FastSearcher fastSearcher = new FastSearcher(backend, - new FS4ResourcePool("container.0", 1), - MockDispatcher.create(Collections.emptyList()), - new SummaryParameters(null), - new ClusterParams("testhittype"), - documentdbInfoConfig); - server.dispatch.packetData = BackendTestCase.PONG; - server.dispatch.setNoChannel(); - Chain chain = new Chain<>(fastSearcher); - Execution e = new Execution(chain, Execution.Context.createContextStub()); - Pong pong = e.ping(new Ping()); - backend.shutdown(); - server.dispatch.socket.close(); - server.dispatch.connection.close(); - server.worker.join(); - pong.setPingInfo("blbl"); - assertEquals("Result of pinging using blbl", pong.toString()); - } - - private void assertCorrectTypes1(FastHit hit) { - assertEquals(String.class, hit.getField("TITLE").getClass()); - assertEquals(Integer.class, hit.getField("BYTES").getClass()); - } - - private void assertCorrectHit1(FastHit hit) { - assertEquals( - "StudyOfMadonna.com - Interviews, Articles, Reviews, Quotes, Essays and more..", - hit.getField("TITLE")); - assertEquals("352", hit.getField("WORDS").toString()); - assertEquals(2003., hit.getRelevance().getScore(), 0.01d); - assertEquals("index:testhittype/234/" + asHexString(hit.getGlobalId()), hit.getId().toString()); - assertEquals("9190", hit.getField("BYTES").toString()); - assertEquals("testhittype", hit.getSource()); - } - - private static String asHexString(GlobalId gid) { - StringBuilder sb = new StringBuilder(); - byte[] rawGid = gid.getRawId(); - for (byte b : rawGid) { - String hex = Integer.toHexString(0xFF & b); - if (hex.length() == 1) - sb.append('0'); - sb.append(hex); - } - return sb.toString(); - } - - @Test - public void null_summary_is_included_in_trace() { - String summary = null; - assertThat(getTraceString(summary), containsString("summary=[null]")); - } - - @Test - public void non_null_summary_is_included_in_trace() { - String summary = "all"; - assertThat(getTraceString(summary), containsString("summary='all'")); - } - - private String getTraceString(String summary) { - FastSearcher fastSearcher = createFastSearcher(); - - Query query = new Query("?query=ignored"); - query.getPresentation().setSummary(summary); - query.setTraceLevel(2); - - Result result = doSearch(fastSearcher, query, 0, 10); - doFill(fastSearcher, result); - - Trace trace = query.getContext(false).getTrace(); - final AtomicReference fillTraceString = new AtomicReference<>(); - - - trace.traceNode().accept(new TraceVisitor() { - @Override - public void visit(TraceNode traceNode) { - if (traceNode.payload() instanceof String && traceNode.payload().toString().contains("fill to dispatch")) - fillTraceString.set((String) traceNode.payload()); - - } - }); - - return fillTraceString.get(); - } - } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java deleted file mode 100644 index 6eab16045c2..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.container.QrSearchersConfig; -import com.yahoo.container.handler.VipStatus; -import com.yahoo.net.HostName; -import com.yahoo.prelude.fastsearch.ClusterParams; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.prelude.fastsearch.FastSearcher; -import com.yahoo.prelude.fastsearch.SummaryParameters; -import com.yahoo.prelude.fastsearch.test.fs4mock.MockBackend; -import com.yahoo.prelude.fastsearch.test.fs4mock.MockFS4ResourcePool; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.dispatch.rpc.MockRpcResourcePoolBuilder; -import com.yahoo.search.dispatch.rpc.RpcResourcePool; -import com.yahoo.search.dispatch.searchcluster.Node; -import com.yahoo.search.searchchain.Execution; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * @author bratseth - */ -class FastSearcherTester { - - public static final String selfHostname = HostName.getLocalhost(); - - private final MockFS4ResourcePool mockFS4ResourcePool; - private final RpcResourcePool mockRpcResourcePool; - private final FastSearcher fastSearcher; - private final MockDispatcher mockDispatcher; - private final VipStatus vipStatus; - - public FastSearcherTester(int containerClusterSize, Node searchNode) { - this(containerClusterSize, Collections.singletonList(searchNode)); - } - - public FastSearcherTester(int containerClusterSize, String... hostAndPortAndGroupStrings) { - this(containerClusterSize, toNodes(hostAndPortAndGroupStrings)); - } - - public FastSearcherTester(int containerClusterSize, List searchNodes) { - String clusterId = "a"; - - var b = new QrSearchersConfig.Builder(); - var searchClusterB = new QrSearchersConfig.Searchcluster.Builder(); - searchClusterB.name(clusterId); - b.searchcluster(searchClusterB); - vipStatus = new VipStatus(b.build()); - - mockFS4ResourcePool = new MockFS4ResourcePool(); - var builder = new MockRpcResourcePoolBuilder(); - searchNodes.forEach(node -> builder.connection(node.key())); - mockRpcResourcePool = builder.build(); - mockDispatcher = MockDispatcher.create(searchNodes, mockFS4ResourcePool, mockRpcResourcePool, containerClusterSize, vipStatus); - fastSearcher = new FastSearcher(new MockBackend(selfHostname, 0L, true), - mockFS4ResourcePool, - mockDispatcher, - new SummaryParameters(null), - new ClusterParams("testhittype"), - new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder())); - } - - private static List toNodes(String... hostAndPortAndGroupStrings) { - List nodes = new ArrayList<>(); - int key = 0; - for (String s : hostAndPortAndGroupStrings) { - String[] parts = s.split(":"); - nodes.add(new Node(key++, parts[0], Integer.parseInt(parts[1]), Integer.parseInt(parts[2]))); - } - return nodes; - } - - public Result search(String query) { - Result result = fastSearcher.search(new Query(query), new Execution(Execution.Context.createContextStub())); - assertEquals(null, result.hits().getError()); - return result; - } - - /** Returns the number of times a backend for this hostname and port has been requested */ - public int requestCount(String hostname, int port) { - return mockFS4ResourcePool.requestCount(hostname, port); - } - - public MockDispatcher dispatcher() { return mockDispatcher; } - - /** Sets the response status of a node and ping it to update the monitor status */ - public void setResponding(String hostname, boolean responding) { - // Start/stop returning a failing backend - mockFS4ResourcePool.setResponding(hostname, responding); - - // Make the search cluster monitor notice right now in this thread - Node node = mockDispatcher.searchCluster().nodesByHost().get(hostname).iterator().next(); - mockDispatcher.searchCluster().ping(node, MoreExecutors.directExecutor()); - } - - /** Sets the response status of a node and ping it to update the monitor status */ - public void setActiveDocuments(String hostname, long activeDocuments) { - mockFS4ResourcePool.setActiveDocuments(hostname, activeDocuments); - - // Make the search cluster monitor notice right now in this thread - Node node = mockDispatcher.searchCluster().nodesByHost().get(hostname).iterator().next(); - mockDispatcher.searchCluster().ping(node, MoreExecutors.directExecutor()); - mockDispatcher.searchCluster().pingIterationCompleted(); - } - - public VipStatus vipStatus() { return vipStatus; } - - /** Retrying is needed because earlier pings from the monitoring thread may interfere with the testing thread */ - public void waitForInRotationIs(boolean expectedRotationStatus) { - int triesLeft = 9000; - while (vipStatus.isInRotation() != expectedRotationStatus && triesLeft > 0) { - triesLeft--; - try { Thread.sleep(10); } catch (InterruptedException e) {} - } - if (triesLeft == 0) - fail("Did not reach VIP in rotation status = " + expectedRotationStatus + " after trying for 90 seconds"); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index ccb265b799b..afb9cf6f571 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -2,8 +2,6 @@ package com.yahoo.prelude.fastsearch.test; import com.yahoo.container.handler.VipStatus; -import com.yahoo.prelude.fastsearch.FS4PingFactory; -import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.RpcInvokerFactory; import com.yahoo.search.dispatch.rpc.RpcResourcePool; @@ -15,23 +13,24 @@ import java.util.List; class MockDispatcher extends Dispatcher { public static MockDispatcher create(List nodes) { - var fs4ResourcePool = new FS4ResourcePool("container.0", 1); var rpcResourcePool = new RpcResourcePool(toDispatchConfig(nodes)); - return create(nodes, fs4ResourcePool, rpcResourcePool, 1, new VipStatus()); + return create(nodes, rpcResourcePool, 1, new VipStatus()); } - public static MockDispatcher create(List nodes, FS4ResourcePool fs4ResourcePool, RpcResourcePool rpcResourcePool, + public static MockDispatcher create(List nodes, RpcResourcePool rpcResourcePool, int containerClusterSize, VipStatus vipStatus) { var dispatchConfig = toDispatchConfig(nodes); var searchCluster = new SearchCluster("a", dispatchConfig, containerClusterSize, vipStatus); - return new MockDispatcher(searchCluster, dispatchConfig, fs4ResourcePool, rpcResourcePool); + return new MockDispatcher(searchCluster, dispatchConfig, rpcResourcePool); } - private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, - RpcResourcePool rpcResourcePool) { - super(searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster, dispatchConfig.dispatchWithProtobuf()), - new FS4PingFactory(fs4ResourcePool), new MockMetric()); + private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcResourcePool rpcResourcePool) { + this(searchCluster, dispatchConfig, new RpcInvokerFactory(rpcResourcePool, searchCluster, dispatchConfig.dispatchWithProtobuf())); + } + + private MockDispatcher(SearchCluster searchCluster, DispatchConfig dispatchConfig, RpcInvokerFactory invokerFactory) { + super(searchCluster, dispatchConfig, invokerFactory, invokerFactory, new MockMetric()); } private static DispatchConfig toDispatchConfig(List nodes) { diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/DispatchThread.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/DispatchThread.java deleted file mode 100644 index d09e8856ee7..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/DispatchThread.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// -*- mode: java; folded-file: t; c-basic-offset: 4 -*- -// -// -package com.yahoo.prelude.fastsearch.test.fs4mock; - - -import com.yahoo.prelude.ConfigurationException; - - -/** - * Thread-wrapper for MockFDispatch - * - * @author Bjorn Borud - */ -public class DispatchThread extends Thread { - int listenPort; - long replyDelay; - long byteDelay; - MockFDispatch dispatch; - Object barrier = new Object(); - - /** - * Instantiate MockFDispatch; if the wanted port is taken we - * bump the port number. Note that the delays are not - * accurate: in reality they will be significantly longer for - * low values. - * - * @param listenPort Wanted port number, note that this may be - * bumped if someone is already running something - * on this port, so it is a starting point for - * scanning only - * @param replyDelay how many milliseconds we should delay when - * replying - * @param byteDelay how many milliseconds we delay for each byte - * written - */ - - public DispatchThread(int listenPort, long replyDelay, long byteDelay) { - this.listenPort = listenPort; - this.replyDelay = replyDelay; - this.byteDelay = byteDelay; - dispatch = new MockFDispatch(listenPort, replyDelay, byteDelay); - dispatch.setBarrier(barrier); - } - - /** - * Run the MockFDispatch and anticipate multiple instances of - * same running. - */ - public void run() { - int maxTries = 20; - // the following section is here to make sure that this - // test is somewhat robust, ie. if someone is already - // listening to the port in question, we'd like to NOT - // fail, but keep probing until we find a port we can use. - boolean up = false; - - while ((!up) && (maxTries-- != 0)) { - try { - dispatch.run(); - up = true; - } catch (ConfigurationException e) { - listenPort++; - dispatch.setListenPort(listenPort); - } - } - } - - /** - * Wait until MockFDispatch is ready to accept connections - * or we time out and indicate which of the two outcomes it was. - * - * @return If we time out we return false. Else we - * return true - * - */ - public boolean waitOnBarrier(long timeout) throws InterruptedException { - long start = System.currentTimeMillis(); - - synchronized (barrier) { - barrier.wait(timeout); - } - long diff = System.currentTimeMillis() - start; - - return (diff < timeout); - } - - /** - * Return the port on which the MockFDispatch actually listens. - * use this instead of assuming where it is since, if more than - * one application tries to use the port we've assigned to it - * we might have to up the port number. - * - * @return port number of active MockFDispatch instance - * - */ - public int listenPort() { - return listenPort; - } -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockBackend.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockBackend.java deleted file mode 100644 index d3fbf8f3645..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockBackend.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test.fs4mock; - -import com.yahoo.fs4.mplex.Backend; -import com.yahoo.fs4.mplex.FS4Channel; - -/** - * @author bratseth - */ -public class MockBackend extends Backend { - - private String hostname; - private final long activeDocumentsInBackend; - private final boolean working; - - /** Created lazily as we want to have just one but it depends on the channel */ - private MockFSChannel channel = null; - - public MockBackend() { - this("", 0L, true); - } - - public MockBackend(String hostname, long activeDocumentsInBackend, boolean working) { - super(); - this.hostname = hostname; - this.activeDocumentsInBackend = activeDocumentsInBackend; - this.working = working; - } - - @Override - public FS4Channel openChannel() { - if (channel == null) - channel = working ? new MockFSChannel(activeDocumentsInBackend, this) - : new NonWorkingMockFSChannel(this); - return channel; - } - - @Override - public FS4Channel openPingChannel() { return openChannel(); } - - @Override - public String getHost() { return hostname; } - - /** Returns the channel in use or null if no channel has been used yet */ - public MockFSChannel getChannel() { return channel; } - - public void shutdown() {} - - @Override - public boolean probeConnection() { - return working; - } -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFDispatch.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFDispatch.java deleted file mode 100644 index 6956f288d1a..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFDispatch.java +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test.fs4mock; - - -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.nio.ByteBuffer; -import java.nio.channels.ClosedByInterruptException; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.ServerSocketChannel; -import java.nio.channels.SocketChannel; -import java.util.HashSet; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; - -import com.yahoo.prelude.ConfigurationException; -import com.yahoo.prelude.fastsearch.test.DocsumDefinitionTestCase; - - -/** - * A server which replies to any query with the same query result after - * a configurable delay, with a configurable slowness (delay between each byte). - * Connections are never timed out. - * - * @author bratseth - */ -public class MockFDispatch { - - private static int connectionCount = 0; - - private static Logger log = Logger.getLogger(MockFDispatch.class.getName()); - - /** The port we accept incoming requests at */ - private int listenPort = 0; - - private long replyDelay; - - private long byteDelay; - - private Object barrier; - - private static byte[] queryResultPacketData = new byte[] { - 0, 0, 0, 64, 0, 0, - 0, 214 - 256, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 0, - 25, 0, 0, 0, 111, 0, 0, 0, 97, 0, 0, 0, 3, 0, 0, 0, 23, 0, 0, 0, 7, 0, 0, - 0, 36, 0, 0, 0, 4, 0, 0, 0, 21, 0, 0, 0, 8, 0, 0, 0, 37}; - - private static byte[] docsumData = DocsumDefinitionTestCase.makeDocsum(); - - private static byte[] docsumHeadPacketData = new byte[] { - 0, 0, 3, 39, 0, 0, - 0, 205 - 256, 0, 0, 0, 1, 0, 0, 0, 0}; - - private static byte[] eolPacketData = new byte[] { - 0, 0, 0, 8, 0, 0, 0, - 200 - 256, 0, 0, 0, 1 }; - - private Set connectionThreads = new HashSet<>(); - - public MockFDispatch(int listenPort, long replyDelay, long byteDelay) { - this.replyDelay = replyDelay; - this.byteDelay = byteDelay; - this.listenPort = listenPort; - } - - public void setBarrier(Object barrier) { - this.barrier = barrier; - } - - public void setListenPort(int listenPort) { - this.listenPort = listenPort; - } - - public void run() { - try { - ServerSocketChannel channel = createServerSocket(listenPort); - - channel.socket().setReuseAddress(true); - while (!Thread.currentThread().isInterrupted()) { - try { - // notify those waiting at the barrier that they - // can now proceed and talk to us - synchronized (barrier) { - if (barrier != null) { - barrier.notify(); - } - } - SocketChannel socketChannel = channel.accept(); - - connectionThreads.add(new ConnectionThread(socketChannel)); - } catch (ClosedByInterruptException e) {// We'll exit - } catch (ClosedChannelException e) { - return; - } catch (Exception e) { - log.log(Level.WARNING, "Unexpected error reading request", e); - } - } - channel.close(); - } catch (IOException e) { - throw new ConfigurationException("Socket channel failure", e); - } - } - - private ServerSocketChannel createServerSocket(int listenPort) - throws IOException { - ServerSocketChannel channel = ServerSocketChannel.open(); - ServerSocket socket = channel.socket(); - - socket.bind( - new InetSocketAddress(InetAddress.getLocalHost(), listenPort)); - String host = socket.getInetAddress().getHostName(); - - log.fine("Accepting dfispatch requests at " + host + ":" + listenPort); - return channel; - } - - public static void main(String[] args) { - log.setLevel(Level.FINE); - MockFDispatch m = new MockFDispatch(7890, Integer.parseInt(args[0]), - Integer.parseInt(args[1])); - - m.run(); - } - - private class ConnectionThread extends Thread { - - private ByteBuffer writeBuffer = ByteBuffer.allocate(2000); - - private ByteBuffer readBuffer = ByteBuffer.allocate(2000); - - private int connectionNr = 0; - - private SocketChannel channel; - - public ConnectionThread(SocketChannel channel) { - this.channel = channel; - fillBuffer(writeBuffer); - start(); - } - - private void fillBuffer(ByteBuffer buffer) { - buffer.clear(); - buffer.put(queryResultPacketData); - buffer.put(docsumHeadPacketData); - buffer.put(docsumData); - buffer.put(docsumHeadPacketData); - buffer.put(docsumData); - buffer.put(eolPacketData); - } - - public void run() { - connectionNr = connectionCount++; - log.fine("Opened connection " + connectionNr); - - try { - long lastRequest = System.currentTimeMillis(); - - while ((System.currentTimeMillis() - lastRequest) <= 5000 - && (!isInterrupted())) { - readBuffer.clear(); - channel.read(readBuffer); - lastRequest = System.currentTimeMillis(); - delay(replyDelay); - - if (byteDelay > 0) { - writeSlow(writeBuffer); - } else { - write(writeBuffer); - } - log.fine( - "Replied in " - + (System.currentTimeMillis() - lastRequest) - + " ms"); - } - - log.fine("Closing timed out connection " + connectionNr); - connectionCount--; - channel.close(); - } catch (IOException e) {} - } - - private void write(ByteBuffer writeBuffer) throws IOException { - writeBuffer.flip(); - channel.write(writeBuffer); - } - - private void writeSlow(ByteBuffer writeBuffer) throws IOException { - writeBuffer.flip(); - int dataSize = writeBuffer.limit(); - - for (int i = 0; i < dataSize; i++) { - writeBuffer.position(i); - writeBuffer.limit(i + 1); - channel.write(writeBuffer); - delay(byteDelay); - } - writeBuffer.limit(dataSize); - } - - private void delay(long delay) { - - try { - Thread.sleep(delay); - } catch (InterruptedException e) {} - } - - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFS4ResourcePool.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFS4ResourcePool.java deleted file mode 100644 index 0d756cbeff3..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFS4ResourcePool.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test.fs4mock; - -import com.yahoo.fs4.mplex.Backend; -import com.yahoo.prelude.fastsearch.FS4ResourcePool; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * @author bratseth - */ -public class MockFS4ResourcePool extends FS4ResourcePool { - - private final Map requestsPerBackend = new HashMap<>(); - private final Set nonRespondingBackends = new HashSet<>(); - private final Map activeDocumentsInBackend = new HashMap<>(); - private final long testingThreadId; - - public MockFS4ResourcePool() { - super("container.0", 1); - this.testingThreadId = Thread.currentThread().getId(); - } - - @Override - public Backend getBackend(String hostname, int port) { - countRequest(hostname + ":" + port); - if (nonRespondingBackends.contains(hostname)) - return new MockBackend(hostname, 0L, false); - else - return new MockBackend(hostname, activeDocumentsInBackend.getOrDefault(hostname, 0L), true); - } - - /** - * Returns the number of times a backend for this hostname and port has been requested - * from the thread creating this - */ - public int requestCount(String hostname, int port) { - return requestsPerBackend.getOrDefault(hostname + ":" + port, 0); - } - - /** Sets the number of active documents the given host will report to have in ping responses */ - public void setActiveDocuments(String hostname, long activeDocuments) { - activeDocumentsInBackend.put(hostname, activeDocuments); - } - - private void countRequest(String hostAndPort) { - // ignore requests from the ping thread to avoid timing issues - if (Thread.currentThread().getId() != testingThreadId) return; - - requestsPerBackend.put(hostAndPort, requestsPerBackend.getOrDefault(hostAndPort, 0) + 1); - } - - public void setResponding(String hostname, boolean responding) { - if (responding) - nonRespondingBackends.remove(hostname); - else - nonRespondingBackends.add(hostname); - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java deleted file mode 100644 index db14a2894db..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/MockFSChannel.java +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test.fs4mock; - -import com.yahoo.document.GlobalId; -import com.yahoo.fs4.BasicPacket; -import com.yahoo.fs4.BufferTooSmallException; -import com.yahoo.fs4.DocumentInfo; -import com.yahoo.fs4.EolPacket; -import com.yahoo.fs4.GetDocSumsPacket; -import com.yahoo.fs4.Packet; -import com.yahoo.fs4.PacketDecoder; -import com.yahoo.fs4.PingPacket; -import com.yahoo.fs4.PongPacket; -import com.yahoo.fs4.QueryPacket; -import com.yahoo.fs4.QueryResultPacket; -import com.yahoo.fs4.mplex.Backend; -import com.yahoo.fs4.mplex.FS4Channel; -import com.yahoo.prelude.fastsearch.test.DocsumDefinitionTestCase; - -import java.nio.ByteBuffer; -import java.util.List; - -/** - * A channel which returns hardcoded packets of the same type as fdispatch - */ -public class MockFSChannel extends FS4Channel { - - /** The number of active documents this should report in ping reponses */ - private final long activeDocuments; - - MockFSChannel(Backend backend) { - this(0, backend); - } - - MockFSChannel(long activeDocuments, Backend backend) { - super(backend, 0); - this.activeDocuments = activeDocuments; - } - - private BasicPacket lastReceived = null; - - private static QueryPacket lastQueryPacket = null; - - /** Initial value of docstamp */ - private static int docstamp = 1088490666; - - private static boolean emptyDocsums = false; - - @Override - public synchronized boolean sendPacket(BasicPacket packet) { - try { - if (packet instanceof Packet) - packet.encode(ByteBuffer.allocate(65536), 0); - } catch (BufferTooSmallException e) { - throw new RuntimeException("Too small buffer to encode packet in mock backend."); - } - - if (packet instanceof QueryPacket) - lastQueryPacket = (QueryPacket) packet; - - lastReceived = packet; - notifyMonitor(); - return true; - } - - /** Change docstamp to invalidate cache */ - public static void resetDocstamp() { - docstamp = 1088490666; - } - - /** Flip sending (in)valid docsums */ - public static void setEmptyDocsums(boolean d) { - emptyDocsums = d; - } - - /** Returns the last query packet received or null if none */ - public QueryPacket getLastQueryPacket() { - return lastQueryPacket; - } - - public BasicPacket getLastReceived() { - return lastReceived; - } - - public BasicPacket[] receivePackets(long timeout, int packetCount) { - List packets = new java.util.ArrayList<>(); - - if (lastReceived instanceof QueryPacket) { - lastQueryPacket = (QueryPacket) lastReceived; - QueryResultPacket result = QueryResultPacket.create(); - - result.setDocstamp(docstamp); - result.setChannel(0); - result.setTotalDocumentCount(2); - result.setOffset(lastQueryPacket.getOffset()); - - if (lastQueryPacket.getOffset() == 0 - && lastQueryPacket.getLastOffset() >= 1) { - result.addDocument( - new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(123), - 2003, 234, 0)); - } - if (lastQueryPacket.getOffset() <= 1 - && lastQueryPacket.getLastOffset() >= 2) { - result.addDocument( - new DocumentInfo(DocsumDefinitionTestCase.createGlobalId(456), - 1855, 234, 1)); - } - packets.add(result); - } - else if (lastReceived instanceof GetDocSumsPacket) { - addDocsums(packets, lastQueryPacket); - } - else if (lastReceived instanceof PingPacket) { - packets.add(new PongPacket(activeDocuments)); - } - while (packetCount >= 0 && packets.size() > packetCount) { - packets.remove(packets.size() - 1); - } - - return packets.toArray(new BasicPacket[packets.size()]); - } - - /** Adds the number of docsums requested in queryPacket.getHits() */ - private void addDocsums(List packets, QueryPacket queryPacket) { - int numHits = queryPacket.getHits(); - - if (lastReceived instanceof GetDocSumsPacket) { - numHits = ((GetDocSumsPacket) lastReceived).getNumDocsums(); - } - for (int i = 0; i < numHits; i++) { - ByteBuffer buffer; - - if (emptyDocsums) { - buffer = createEmptyDocsumPacketData(); - } else { - int[] docids = { - 123, 456, 789, 789, 789, 789, 789, 789, 789, - 789, 789, 789 }; - - buffer = createDocsumPacketData(docids[i], DocsumDefinitionTestCase.makeDocsum()); - } - buffer.position(0); - packets.add(PacketDecoder.decode(buffer)); - } - packets.add(EolPacket.create()); - } - - private ByteBuffer createEmptyDocsumPacketData() { - ByteBuffer buffer = ByteBuffer.allocate(16); - - buffer.limit(buffer.capacity()); - buffer.position(0); - buffer.putInt(12); // length - buffer.putInt(205); // a code for docsumpacket - buffer.putInt(0); // channel - buffer.putInt(0); // dummy location - return buffer; - } - - private ByteBuffer createDocsumPacketData(int docid, byte[] docsumData) { - ByteBuffer buffer = ByteBuffer.allocate(docsumData.length + 4 + 8 + GlobalId.LENGTH); - - buffer.limit(buffer.capacity()); - buffer.position(0); - buffer.putInt(docsumData.length + 8 + GlobalId.LENGTH); - buffer.putInt(205); // Docsum packet code - buffer.putInt(0); - byte[] rawGid = DocsumDefinitionTestCase.createGlobalId(docid).getRawId(); - buffer.put(rawGid); - buffer.put(docsumData); - return buffer; - } - - public void close() {} -} diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/NonWorkingMockFSChannel.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/NonWorkingMockFSChannel.java deleted file mode 100644 index c7425afd611..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/fs4mock/NonWorkingMockFSChannel.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.fastsearch.test.fs4mock; - -import com.yahoo.fs4.BasicPacket; -import com.yahoo.fs4.mplex.Backend; - -/** - * @author bratseth - */ -public class NonWorkingMockFSChannel extends MockFSChannel { - - public NonWorkingMockFSChannel(Backend backend) { - super(backend); - } - - @Override - public synchronized boolean sendPacket(BasicPacket bPacket) { - return false; - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 42a22f6f86b..7ee62ae9978 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -88,11 +88,6 @@ public class MockSearchCluster extends SearchCluster { } } - @Override - public ImmutableMultimap nodesByHost() { - return nodesByHost; - } - @Override public Optional directDispatchTarget() { return Optional.empty(); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java index 6ed8a209cc5..7bdf1916d85 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java @@ -3,7 +3,6 @@ package com.yahoo.search.grouping.vespa; import com.yahoo.document.DocumentId; import com.yahoo.document.GlobalId; -import com.yahoo.fs4.QueryPacketData; import com.yahoo.net.URI; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; @@ -61,19 +60,6 @@ public class HitConverterTestCase { assertEquals(ctxHit.getSource(), hit.getSource()); } - @Test - public void requireThatHitTagIsCopiedFromGroupingListContext() { - QueryPacketData ctxTag = new QueryPacketData(); - GroupingListHit ctxHit = context(); - ctxHit.setQueryPacketData(ctxTag); - - HitConverter converter = new HitConverter(new MySearcher(), new Query()); - Hit hit = converter.toSearchHit("default", new FS4Hit(1, createGlobalId(2), 3).setContext(ctxHit)); - assertNotNull(hit); - assertTrue(hit instanceof FastHit); - assertSame(ctxTag, ((FastHit)hit).getQueryPacketData()); - } - @Test public void requireThatSummaryClassIsSet() { Searcher searcher = new MySearcher(); diff --git a/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java new file mode 100644 index 00000000000..8aff81a90db --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/RankFeaturesTestCase.java @@ -0,0 +1,140 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.io.GrowableByteBuffer; +import com.yahoo.search.query.ranking.RankFeatures; +import com.yahoo.search.query.ranking.RankProperties; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import com.yahoo.text.Utf8; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; + +/** + * @author geirst + */ +public class RankFeaturesTestCase { + + @Test + public void requireThatRankPropertiesTakesBothStringAndObject() { + RankProperties p = new RankProperties(); + p.put("string", "b"); + p.put("object", 7); + assertEquals("7", p.get("object").get(0)); + assertEquals("b", p.get("string").get(0)); + } + + @Test + public void requireThatRankFeaturesUsingDoubleAndDoubleToStringEncodeTheSameWay() { + RankFeatures withDouble = new RankFeatures(); + withDouble.put("query(myDouble)", 3.8); + assertEquals(3.8, withDouble.getDouble("query(myDouble)").getAsDouble(), 0.000001); + + RankFeatures withString = new RankFeatures(); + withString.put("query(myDouble)", String.valueOf(3.8)); + + RankProperties withDoubleP = new RankProperties(); + withDouble.prepare(withDoubleP); + RankProperties withStringP = new RankProperties(); + withString.prepare(withStringP); + + byte[] withDoubleEncoded = encode(withDoubleP); + byte[] withStringEncoded = encode(withStringP); + assertEquals(Arrays.toString(withStringEncoded), Arrays.toString(withDoubleEncoded)); + } + + @Test + public void requireThatSingleTensorIsBinaryEncoded() { + TensorType type = new TensorType.Builder().mapped("x").mapped("y").mapped("z").build(); + Tensor tensor = Tensor.from(type, "{ {x:a, y:b, z:c}:2.0, {x:a, y:b, z:c2}:3.0 }"); + assertTensorEncodingAndDecoding(type, "query(my_tensor)", "my_tensor", tensor); + assertTensorEncodingAndDecoding(type, "$my_tensor", "my_tensor", tensor); + } + + @Test + public void requireThatMultipleTensorsAreBinaryEncoded() { + TensorType type = new TensorType.Builder().mapped("x").mapped("y").mapped("z").build(); + Tensor tensor1 = Tensor.from(type, "{ {x:a, y:b, z:c}:2.0, {x:a, y:b, z:c2}:3.0 }"); + Tensor tensor2 = Tensor.from(type, "{ {x:a, y:b, z:c}:5.0 }"); + assertTensorEncodingAndDecoding(type, Arrays.asList( + new Entry("query(tensor1)", "tensor1", tensor1), + new Entry("$tensor2", "tensor2", tensor2))); + } + + private static class Entry { + final String key; + final String normalizedKey; + final Tensor tensor; + Entry(String key, String normalizedKey, Tensor tensor) { + this.key = key; + this.normalizedKey = normalizedKey; + this.tensor = tensor; + } + } + + private static void assertTensorEncodingAndDecoding(TensorType type, List entries) { + RankProperties properties = createRankPropertiesWithTensors(entries); + assertEquals(entries.size(), properties.asMap().size()); + + Map decodedProperties = decode(type, encode(properties)); + assertEquals(entries.size(), properties.asMap().size()); + assertEquals(entries.size(), decodedProperties.size()); + for (Entry entry : entries) { + assertEquals(entry.tensor, decodedProperties.get(entry.normalizedKey)); + } + } + + private static void assertTensorEncodingAndDecoding(TensorType type, String key, String normalizedKey, Tensor tensor) { + assertTensorEncodingAndDecoding(type, Arrays.asList(new Entry(key, normalizedKey, tensor))); + } + + private static RankProperties createRankPropertiesWithTensors(List entries) { + RankFeatures features = new RankFeatures(); + for (Entry entry : entries) { + features.put(entry.key, entry.tensor); + } + RankProperties properties = new RankProperties(); + features.prepare(properties); + return properties; + } + + private static byte[] encode(RankProperties properties) { + ByteBuffer buffer = ByteBuffer.allocate(512); + properties.encode(buffer, true); + byte[] result = new byte[buffer.position()]; + buffer.rewind(); + buffer.get(result); + return result; + } + + private static Map decode(TensorType type, byte[] encodedProperties) { + GrowableByteBuffer buffer = GrowableByteBuffer.wrap(encodedProperties); + byte[] mapNameBytes = new byte[buffer.getInt()]; + buffer.get(mapNameBytes); + int numEntries = buffer.getInt(); + Map result = new HashMap<>(); + for (int i = 0; i < numEntries; ++i) { + byte[] keyBytes = new byte[buffer.getInt()]; + buffer.get(keyBytes); + String key = Utf8.toString(keyBytes); + byte[] value = new byte[buffer.getInt()]; + buffer.get(value); + if (key.contains(".type")) { + result.put(key, Utf8.toString(value)); + } else { + result.put(key, TypedBinaryFormat.decode(Optional.of(type), GrowableByteBuffer.wrap(value))); + } + } + return result; + } + +} -- cgit v1.2.3