diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /container-search/src/test/java/com/yahoo/search |
Publish
Diffstat (limited to 'container-search/src/test/java/com/yahoo/search')
370 files changed, 32402 insertions, 0 deletions
diff --git a/container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java b/container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java new file mode 100644 index 00000000000..6c3f1eba4c0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/StupidSingleThreadedHttpServer.java @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search; + +import com.yahoo.text.Utf8; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * As the name implies, a stupid, single-threaded bad-excuse-for-HTTP server. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class StupidSingleThreadedHttpServer implements Runnable { + + private static final Logger log = Logger.getLogger(StupidSingleThreadedHttpServer.class.getName()); + + private final ServerSocket serverSocket; + private final int delaySeconds; + private Thread serverThread = null; + private CompletableFuture<String> requestFuture = new CompletableFuture<>(); + private final Pattern contentLengthPattern = Pattern.compile("content-length: (\\d+)", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL | Pattern.MULTILINE); + + public StupidSingleThreadedHttpServer() throws IOException { + this(0, 0); + } + + public StupidSingleThreadedHttpServer(int port, int delaySeconds) throws IOException { + this.delaySeconds = delaySeconds; + this.serverSocket = new ServerSocket(port); + } + + public void start() { + serverThread = new Thread(this); + serverThread.setDaemon(true); + serverThread.start(); + } + + public void run() { + try { + while(true) { + Socket socket = serverSocket.accept(); + StringBuilder request = new StringBuilder(); + socket.setSoLinger(true, 60); + BufferedReader in = new BufferedReader( + new InputStreamReader( + socket.getInputStream())); + + int contentLength = -1; + String inputLine; + while (!"".equals(inputLine = in.readLine())) { //read header: + request.append(inputLine).append("\r\n"); + if (inputLine.toLowerCase(Locale.US).contains("content-length")) { + Matcher contentLengthMatcher = contentLengthPattern.matcher(inputLine); + if (contentLengthMatcher.matches()) { + contentLength = Integer.parseInt(contentLengthMatcher.group(1)); + } + } + } + request.append("\r\n"); + + if (contentLength < 0) { + System.err.println("WARNING! Got no Content-Length header!!"); + } else { + char[] requestBody = new char[contentLength]; + int readRemaining = contentLength; + + do { + int read = in.read(requestBody, (contentLength - readRemaining), readRemaining); + if (read < 0) { + throw new IllegalStateException("Should not get EOF here!!"); + } + readRemaining -= read; + } while (readRemaining > 0); + + request.append(new String(requestBody)); + } + + // Simulate service slowness + if (delaySeconds > 0) { + try { + System.out.println(this.getClass().getCanonicalName() + " sleeping in " + delaySeconds + " s before responding..."); + Thread.sleep((long) (delaySeconds * 1000)); + System.out.println("done sleeping, responding"); + } catch (InterruptedException e) { + //ignore + } + } + + socket.getOutputStream().write(getResponse(request.toString())); + socket.getOutputStream().flush(); + in.close(); + socket.close(); + + boolean wasCompleted = requestFuture.complete(request.toString()); + if (!wasCompleted) { + log.log(Level.INFO, "Only the first request will be stored, ignoring. " + + "Old value: " + requestFuture.get() + + ", New value: " + request.toString()); + } + } + } catch (SocketException se) { + if ("Socket closed".equals(se.getMessage())) { + //ignore + } else { + throw new RuntimeException(se); + } + } catch (IOException|InterruptedException|ExecutionException e) { + throw new RuntimeException(e); + } + } + + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 0\r\n" + + "\r\n"); + } + + protected byte[] getResponseBody() { + return new byte[0]; + } + + public void stop() { + if (!serverSocket.isClosed()) { + try { + serverSocket.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + try { + serverThread.interrupt(); + } catch (Exception e) { + //ignore + } + } + + public int getServerPort() { + return serverSocket.getLocalPort(); + } + + public String getRequest() { + try { + return requestFuture.get(1, TimeUnit.MINUTES); + } catch (InterruptedException | ExecutionException | TimeoutException e) { + throw new AssertionError("Failed waiting for request. ", e); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java new file mode 100644 index 00000000000..01392e900d8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusterSearcherTestCase.java @@ -0,0 +1,169 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.cluster.test; + +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.cluster.ClusterSearcher; +import com.yahoo.search.cluster.Hasher; +import com.yahoo.search.cluster.PingableSearcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +// TODO: Author! +public class ClusterSearcherTestCase extends TestCase { + + + class TestingBackendSearcher extends PingableSearcher { + + Hit hit; + + public TestingBackendSearcher(Hit hit) { + this.hit = hit; + } + + public @Override Result search(Query query,Execution execution) { + Result result = execution.search(query); + result.hits().add(hit); + return result; + } + } + + class BlockingBackendSearcher extends TestingBackendSearcher { + + private boolean blocking = false; + + public BlockingBackendSearcher(Hit hit) { + super(hit); + } + + public Result search(Query query,Execution execution) { + Result result = super.search(query,execution); + if(blocking) { + result.hits().addError(ErrorMessage.createUnspecifiedError("Dummy error")); + } + return result; + } + + @Override + public Pong ping(Ping ping, Execution execution) { + //Sleep an hour + Pong pong = new Pong(); + if (isBlocking()) { + + pong.addError(ErrorMessage.createTimeout("Dummy timeout")); + } + return new Pong(); + } + + public boolean isBlocking() { + return blocking; + } + + public void setBlocking(boolean blocking) { + this.blocking = blocking; + } + } + + class SimpleQuery extends Query { + int hashValue; + public SimpleQuery(int hashValue) { + this.hashValue = hashValue; + } + + @Override + public int hashCode() { + return hashValue; + } + } + + class SimpleHasher<T> extends Hasher<T> { + + + class SimpleNodeList extends NodeList<T> { + public SimpleNodeList() { + super(null); + } + + public T select(int code, int trynum) { + return objects.get(code + trynum % objects.size()); + } + + public int getNodeCount() { + return objects.size(); + } + } + + List<T> objects = new ArrayList<>(); + + @Override + public synchronized void remove(T node) { + objects.remove(node); + } + + @Override + public synchronized void add(T node) { + objects.add(node); + } + + @Override + public NodeList<T> getNodes() { + return new SimpleNodeList(); + + } + } + + /** A cluster searcher which clusters over a set of alternative searchers (search chains would be more realistic) */ + static class SearcherClusterSearcher extends ClusterSearcher<Searcher> { + + public SearcherClusterSearcher(ComponentId id,List<Searcher> searchers,Hasher<Searcher> hasher) { + super(id,searchers,hasher,false); + } + + public @Override Result search(Query query,Execution execution,Searcher searcher) { + return searcher.search(query,execution); + } + + public @Override void fill(Result result,String summaryName,Execution execution,Searcher searcher) { + searcher.fill(result,summaryName,execution); + } + + public @Override Pong ping(Ping ping,Searcher searcher) { + return new Execution(searcher, Execution.Context.createContextStub()).ping(ping); + } + + } + + + public void testSimple() { + Hit blockingHit = new Hit("blocking"); + Hit nonblockingHit = new Hit("nonblocking"); + BlockingBackendSearcher blockingSearcher = new BlockingBackendSearcher(blockingHit); + List<Searcher> searchers=new ArrayList<>(); + searchers.add(blockingSearcher); + searchers.add(new TestingBackendSearcher(nonblockingHit)); + ClusterSearcher<?> provider = new SearcherClusterSearcher(new ComponentId("simple"),searchers,new SimpleHasher<>()); + + Result blockingResult = new Execution(provider, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals(blockingHit,blockingResult.hits().get(0)); + Result nonblockingResult = new Execution(provider, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals(nonblockingHit,nonblockingResult.hits().get(0)); + + blockingSearcher.setBlocking(true); + + blockingResult = new Execution(provider, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals(blockingResult.hits().get(0),nonblockingHit); + nonblockingResult = new Execution(provider, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals(nonblockingResult.hits().get(0),nonblockingHit); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java new file mode 100644 index 00000000000..06686e8777a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -0,0 +1,198 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.cluster.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.concurrent.DaemonThreadFactory; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.cluster.ClusterSearcher; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; + +/** + * @author bratseth + */ +public class ClusteredConnectionTestCase extends junit.framework.TestCase { + + public void testClustering() { + Connection connection0=new Connection("0"); + Connection connection1=new Connection("1"); + Connection connection2=new Connection("2"); + List<Connection> connections=new ArrayList<>(); + connections.add(connection0); + connections.add(connection1); + connections.add(connection2); + MyBackend myBackend=new MyBackend(new ComponentId("test"),connections); + + Result r; + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:2",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:1",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + + connection2.setInService(false); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:1",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + + connection1.setInService(false); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + + connection0.setInService(false); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("Failed calling connection '2' in searcher 'test' for query 'NULL': Connection failed", + r.hits().getError().getDetailedMessage()); + + connection0.setInService(true); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + + connection1.setInService(true); + connection2.setInService(true); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(1)); + assertEquals("from:2",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(2)); + assertEquals("from:1",r.hits().get(0).getId().stringValue()); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(3)); + assertEquals("from:0",r.hits().get(0).getId().stringValue()); + } + + public void testClusteringWithPing() { + Connection connection0=new Connection("0"); + Connection connection1=new Connection("1"); + Connection connection2=new Connection("2"); + List<Connection> connections=new ArrayList<>(); + connections.add(connection0); + connections.add(connection1); + connections.add(connection2); + MyBackend myBackend=new MyBackend(new ComponentId("test"),connections); + + Result r; + + // Note that we cannot make any successful queries here or we have to wait 10 seconds for + // the traffic monitor to agree that these nodes are really not responding + + connection2.setInService(false); + connection1.setInService(false); + connection0.setInService(false); + forcePing(myBackend); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertEquals("No backends in service. Try later",r.hits().getError().getMessage()); + + connection2.setInService(true); + connection1.setInService(true); + connection0.setInService(true); + forcePing(myBackend); + r=new Execution(myBackend, Execution.Context.createContextStub()).search(new SimpleQuery(0)); + assertNull(r.hits().getError()); + } + + private void forcePing(MyBackend myBackend) { + myBackend.getMonitor().ping(Executors.newCachedThreadPool(new DaemonThreadFactory())); + Thread.yield(); + } + + /** Represents a connection, e.g over http, in this test */ + private static class Connection { + + private String id; + + private boolean inService=true; + + public Connection(String id) { + this.id=id; + } + + /** This is used for both fill, pings and queries */ + public String getResponse() { + if (!inService) throw new RuntimeException("Connection failed"); + return id; + } + + public void setInService(boolean inservice) { + this.inService=inservice; + } + + public String toString() { + return "connection '" + id + "'"; + } + + } + + /** + * This is the kind of searcher which will be implemented by those who wish to create a searcher which is a + * client to a clustered service. + * The goal is to make writing this correctly as simple as possible. + */ + private static class MyBackend extends ClusterSearcher<Connection> { + + public MyBackend(ComponentId componentId, List<Connection> connections) { + super(componentId,connections,false); + } + + public @Override Result search(Query query,Execution execution,Connection connection) { + Result result=new Result(query); + result.hits().add(new Hit("from:" + connection.getResponse())); + return result; + } + + public @Override void fill(Result result,String summary,Execution execution,Connection connection) { + result.hits().get(0).fields().put("filled",connection.getResponse()); + } + + public @Override Pong ping(Ping ping,Connection connection) { + Pong pong=new Pong(); + if (connection.getResponse()==null) + pong.addError(ErrorMessage.createBackendCommunicationError("No ping response from '" + connection + "'")); + return pong; + } + + } + + /** A query with a predictable hash function */ + private static class SimpleQuery extends Query { + + int hashValue; + + public SimpleQuery(int hashValue) { + this.hashValue = hashValue; + } + + public @Override int hashCode() { + return hashValue; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/debug/test/SearchChainTextRepresentationTestCase.java b/container-search/src/test/java/com/yahoo/search/debug/test/SearchChainTextRepresentationTestCase.java new file mode 100644 index 00000000000..e952300e0ed --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/debug/test/SearchChainTextRepresentationTestCase.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.debug.test; + +import junit.framework.TestCase; + +import com.yahoo.search.debug.SearchChainTextRepresentation; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.test.SimpleSearchChain; + +/** + * Test of SearchChainTextRepresentation. + * @author tonytv + */ +public class SearchChainTextRepresentationTestCase extends TestCase { + + public void testTextRepresentation() { + SearchChainTextRepresentation textRepresentation = + new SearchChainTextRepresentation(SimpleSearchChain.orderedChain, new SearchChainRegistry()); + + String[] expected = { + "test [Searchchain] {", + " one [Searcher] {", + " Reason for forwarding to this search chain.", + " child-chain [Searchchain] {", + " child-searcher [Searcher]", + " }", + " child-chain [Searchchain] {", + " child-searcher [Searcher]", + " }", + " }", + " two [Searcher] {", + " Reason for forwarding to this search chain.", + " child-chain [Searchchain] {", + " child-searcher [Searcher]", + " }", + " child-chain [Searchchain] {", + " child-searcher [Searcher]", + " }", + " }", + "}" + }; + + String[] result = textRepresentation.toString().split("\n"); + assertEquals(expected.length, result.length); + + int i = 0; + for (String line : textRepresentation.toString().split("\n")) + assertEquals(expected[i++], line); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java b/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java new file mode 100644 index 00000000000..a88ef7e5e37 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/FillTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import com.yahoo.compress.CompressionType; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.Map; + + +/** + * Tests using a dispatcher to fill a result + * + * @author bratseth + */ +public class FillTestCase { + + private MockClient client = new MockClient(); + + @Test + public void testFilling() { + Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); + nodes.put(0, client.createConnection("host0", 123)); + nodes.put(1, client.createConnection("host1", 123)); + nodes.put(2, client.createConnection("host2", 123)); + Dispatcher dispatcher = new Dispatcher(nodes, client); + + Query query = new Query(); + Result result = new Result(query); + result.hits().add(createHit(0, 0)); + result.hits().add(createHit(2, 1)); + result.hits().add(createHit(1, 2)); + result.hits().add(createHit(2, 3)); + result.hits().add(createHit(0, 4)); + + client.setDocsumReponse("host0", 0, "summaryClass1", map("field1", "s.0.0", "field2", 0)); + client.setDocsumReponse("host2", 1, "summaryClass1", map("field1", "s.2.1", "field2", 1)); + client.setDocsumReponse("host1", 2, "summaryClass1", map("field1", "s.1.2", "field2", 2)); + client.setDocsumReponse("host2", 3, "summaryClass1", map("field1", "s.2.3", "field2", 3)); + client.setDocsumReponse("host0", 4, "summaryClass1", map("field1", "s.0.4", "field2", 4)); + dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); + + assertEquals("s.0.0", result.hits().get("hit:0").getField("field1").toString()); + assertEquals("s.2.1", result.hits().get("hit:1").getField("field1").toString()); + assertEquals("s.1.2", result.hits().get("hit:2").getField("field1").toString()); + assertEquals("s.2.3", result.hits().get("hit:3").getField("field1").toString()); + assertEquals("s.0.4", result.hits().get("hit:4").getField("field1").toString()); + assertEquals(0L, result.hits().get("hit:0").getField("field2")); + assertEquals(1L, result.hits().get("hit:1").getField("field2")); + assertEquals(2L, result.hits().get("hit:2").getField("field2")); + assertEquals(3L, result.hits().get("hit:3").getField("field2")); + assertEquals(4L, result.hits().get("hit:4").getField("field2")); + } + + @Test + public void testErrorHandling() { + client.setMalfunctioning(true); + + Map<Integer, Client.NodeConnection> nodes = new HashMap<>(); + nodes.put(0, client.createConnection("host0", 123)); + Dispatcher dispatcher = new Dispatcher(nodes, client); + + Query query = new Query(); + Result result = new Result(query); + result.hits().add(createHit(0, 0)); + + dispatcher.fill(result, "summaryClass1", CompressionType.valueOf("LZ4")); + + assertEquals("Malfunctioning", result.hits().getError().getDetailedMessage()); + } + + private FastHit createHit(int sourceNodeId, int hitId) { + FastHit hit = new FastHit("hit:" + hitId, 1.0); + hit.setPartId(sourceNodeId, 0); + hit.setDistributionKey(sourceNodeId); + hit.setGlobalId(client.globalIdFrom(hitId)); + return hit; + } + + private Map<String, Object> map(String stringKey, String stringValue, String intKey, int intValue) { + Map<String, Object> map = new HashMap<>(); + map.put(stringKey, stringValue); + map.put(intKey, intValue); + return map; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockClient.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockClient.java new file mode 100644 index 00000000000..2a7301652b9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockClient.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import com.yahoo.compress.CompressionType; +import com.yahoo.compress.Compressor; +import com.yahoo.document.GlobalId; +import com.yahoo.document.idstring.IdIdString; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.BinaryFormat; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author bratseth + */ +public class MockClient implements Client { + + private final Map<DocsumKey, Map<String, Object>> docsums = new HashMap<>(); + private final Compressor compressor = new Compressor(); + private boolean malfunctioning = false; + + /** Set to true to cause this to produce an error instead of a regular response */ + public void setMalfunctioning(boolean malfunctioning) { this.malfunctioning = malfunctioning; } + + @Override + public NodeConnection createConnection(String hostname, int port) { + return new MockNodeConnection(hostname, port); + } + + @Override + public void getDocsums(List<FastHit> hitsContext, NodeConnection node, CompressionType compression, + int uncompressedSize, byte[] compressedSlime, Dispatcher.GetDocsumsResponseReceiver responseReceiver, + double timeoutSeconds) { + if (malfunctioning) { + responseReceiver.receive(GetDocsumsResponseOrError.fromError("Malfunctioning")); + return; + } + + Inspector request = BinaryFormat.decode(compressor.decompress(compressedSlime, compression, uncompressedSize)).get(); + String docsumClass = request.field("class").asString(); + List<Map<String, Object>> docsumsToReturn = new ArrayList<>(); + request.field("gids").traverse((ArrayTraverser)(index, gid) -> { + GlobalId docId = new GlobalId(gid.asData()); + docsumsToReturn.add(docsums.get(new DocsumKey(node.toString(), docId, docsumClass))); + }); + Slime responseSlime = new Slime(); + Cursor root = responseSlime.setObject(); + Cursor docsums = root.setArray("docsums"); + for (Map<String, Object> docsumFields : docsumsToReturn) { + Cursor docsumItem = docsums.addObject(); + Cursor docsum = docsumItem.setObject("docsum"); + for (Map.Entry<String, Object> field : docsumFields.entrySet()) { + if (field.getValue() instanceof Integer) + docsum.setLong(field.getKey(), (Integer)field.getValue()); + else if (field.getValue() instanceof String) + docsum.setString(field.getKey(), (String)field.getValue()); + else + throw new RuntimeException(); + } + } + byte[] slimeBytes = BinaryFormat.encode(responseSlime); + Compressor.Compression compressionResult = compressor.compress(compression, slimeBytes); + GetDocsumsResponse response = new GetDocsumsResponse(compressionResult.type().getCode(), slimeBytes.length, + compressionResult.data(), hitsContext); + responseReceiver.receive(GetDocsumsResponseOrError.fromResponse(response)); + } + + public void setDocsumReponse(String nodeId, int docId, String docsumClass, Map<String, Object> docsumValues) { + docsums.put(new DocsumKey(nodeId, globalIdFrom(docId), docsumClass), docsumValues); + } + + public GlobalId globalIdFrom(int hitId) { + return new GlobalId(new IdIdString("", "test", "", String.valueOf(hitId))); + } + + private static class MockNodeConnection implements Client.NodeConnection { + + private final String hostname; + + public MockNodeConnection(String hostname, int port) { + this.hostname = hostname; + } + + @Override + public void close() { } + + @Override + public String toString() { return hostname; } + + } + + private static class DocsumKey { + + private final String internalKey; + + public DocsumKey(String nodeId, GlobalId docId, String docsumClass) { + internalKey = docsumClass + "." + nodeId + "." + docId; + } + + @Override + public int hashCode() { return internalKey.hashCode(); } + + @Override + public boolean equals(Object other) { + if ( ! (other instanceof DocsumKey)) return false; + return ((DocsumKey)other).internalKey.equals(this.internalKey); + } + + @Override + public String toString() { return internalKey; } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java b/container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java new file mode 100644 index 00000000000..37969e12399 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/FutureWaiterTest.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation; + + +/** + * @author tonytv + */ +// TODO: Fix or remove! +public class FutureWaiterTest { + +/* + + @MockClass(realClass = System.class) + public static class MockSystem { + + private static long currentTime; + private static boolean firstTime; + + private static final long startTime = 123; + + @Mock + public static synchronized long currentTimeMillis() { + if (firstTime) { + firstTime = false; + return startTime; + } + return currentTime; + } + + static synchronized void setElapsedTime(long elapsedTime) { + firstTime = true; + currentTime = elapsedTime + startTime; + } + } + + @Mocked() + FutureResult result1; + + @Mocked() + FutureResult result2; + + @Mocked() + FutureResult result3; + + @Mocked() + FutureResult result4; + + @Before + public void before() { + Mockit.setUpMock(FutureWaiterTest.MockSystem.class); + } + + @After + public void after() { + Mockit.tearDownMocks(); + } + + @Test + public void require_time_to_wait_is_adjusted_for_elapsed_time() { + MockSystem.setElapsedTime(300); + + FutureWaiter futureWaiter = new FutureWaiter(); + futureWaiter.add(result1, 350); + futureWaiter.waitForFutures(); + + new FullVerifications() { + { + result1.get(350 - 300, TimeUnit.MILLISECONDS); + } + }; + } + + @Test + public void require_do_not_wait_for_expired_timeouts() { + MockSystem.setElapsedTime(300); + + FutureWaiter futureWaiter = new FutureWaiter(); + futureWaiter.add(result1, 300); + futureWaiter.add(result2, 290); + + futureWaiter.waitForFutures(); + + new FullVerifications() { + {} + }; + } + + @Test + public void require_wait_for_largest_timeout_first() throws InterruptedException { + MockSystem.setElapsedTime(600); + + FutureWaiter futureWaiter = new FutureWaiter(); + futureWaiter.add(result1, 500); + futureWaiter.add(result4, 800); + futureWaiter.add(result2, 600); + futureWaiter.add(result3, 700); + + futureWaiter.waitForFutures(); + + new FullVerifications() { + { + result4.get(800 - 600, TimeUnit.MILLISECONDS); + result3.get(700 - 600, TimeUnit.MILLISECONDS); + } + }; + } + + */ +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java new file mode 100644 index 00000000000..c707702a3d3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java @@ -0,0 +1,212 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import static org.junit.Assert.*; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.Random; +import java.util.zip.GZIPOutputStream; + +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.message.BasicHeader; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.text.Utf8; + +/** + * Test GZip support for the HTTP integration introduced in 4.2. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class GzipDecompressingEntityTestCase { + private static final String STREAM_CONTENT = "00000000000000000000000000000000000000000000000000"; + private static final byte[] CONTENT_AS_BYTES = Utf8.toBytes(STREAM_CONTENT); + GzipDecompressingEntity testEntity; + + private static final class MockEntity implements HttpEntity { + + private final InputStream inStream; + + MockEntity(InputStream is) { + inStream = is; + } + + @Override + public boolean isRepeatable() { + return false; + } + + @Override + public boolean isChunked() { + return false; + } + + @Override + public long getContentLength() { + return -1; + } + + @Override + public Header getContentType() { + return new BasicHeader("Content-Type", "text/plain"); + } + + @Override + public Header getContentEncoding() { + return new BasicHeader("Content-Encoding", "gzip"); + } + + @Override + public InputStream getContent() throws IOException, + IllegalStateException { + return inStream; + } + + @Override + public void writeTo(OutputStream outstream) throws IOException { + } + + @Override + public boolean isStreaming() { + return false; + } + + @Override + public void consumeContent() throws IOException { + } + } + + @Before + public void setUp() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(out); + gzip.write(CONTENT_AS_BYTES); + gzip.finish(); + gzip.close(); + byte[] compressed = out.toByteArray(); + InputStream inStream = new ByteArrayInputStream(compressed); + testEntity = new GzipDecompressingEntity(new MockEntity(inStream)); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetContentLength() throws UnknownHostException { + assertEquals(STREAM_CONTENT.length(), testEntity.getContentLength()); + } + + @Test + public final void testGetContent() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length]; + int read = in.read(buffer); + assertEquals(CONTENT_AS_BYTES.length, read); + assertArrayEquals(CONTENT_AS_BYTES, buffer); + } + + @Test + public final void testGetContentToBigArray() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; + in.read(buffer); + byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); + assertArrayEquals(expected, buffer); + } + + @Test + public final void testGetContentAvailable() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + assertEquals(CONTENT_AS_BYTES.length, in.available()); + } + + @Test + public final void testLargeZip() throws IOException { + byte [] input = new byte [10000000]; + Random random = new Random(89); + for (int i = 0; i < input.length; i++) { + input[i] = (byte) random.nextInt(); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(out); + gzip.write(input); + gzip.finish(); + gzip.close(); + byte[] compressed = out.toByteArray(); + assertEquals(10003073, compressed.length); + InputStream inStream = new ByteArrayInputStream(compressed); + GzipDecompressingEntity gunzipper = new GzipDecompressingEntity(new MockEntity(inStream)); + assertEquals(input.length, gunzipper.getContentLength()); + byte[] buffer = new byte[input.length]; + InputStream content = gunzipper.getContent(); + assertEquals(input.length, content.available()); + int read = content.read(buffer); + assertEquals(input.length, read); + assertArrayEquals(input, buffer); + } + + @Test + public final void testGetContentReadByte() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; + int i = 0; + while (i < buffer.length) { + int r = in.read(); + if (r == -1) { + break; + } else { + buffer[i++] = (byte) r; + } + } + byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); + assertEquals(CONTENT_AS_BYTES.length, i); + assertArrayEquals(expected, buffer); + } + + @Test + public final void testGetContentReadWithOffset() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; + int read = in.read(buffer, CONTENT_AS_BYTES.length, CONTENT_AS_BYTES.length); + assertEquals(CONTENT_AS_BYTES.length, read); + byte[] expected = new byte[CONTENT_AS_BYTES.length * 2]; + for (int i = 0; i < CONTENT_AS_BYTES.length; ++i) { + expected[CONTENT_AS_BYTES.length + i] = CONTENT_AS_BYTES[i]; + } + assertArrayEquals(expected, buffer); + read = in.read(buffer, 0, CONTENT_AS_BYTES.length); + assertEquals(-1, read); + } + + @Test + public final void testGetContentSkip() throws IllegalStateException, IOException { + InputStream in = testEntity.getContent(); + final long n = 5L; + long skipped = in.skip(n); + assertEquals(n, skipped); + int read = in.read(); + assertEquals(CONTENT_AS_BYTES[(int) n], read); + skipped = in.skip(5000); + assertEquals(CONTENT_AS_BYTES.length - n - 1, skipped); + assertEquals(-1L, in.skip(1L)); + } + + + @Test + public final void testWriteToOutputStream() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + testEntity.writeTo(out); + assertArrayEquals(CONTENT_AS_BYTES, out.toByteArray()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java new file mode 100644 index 00000000000..c3bd2ada260 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java @@ -0,0 +1,238 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.search.federation.ProviderConfig; +import org.junit.Test; + +import static com.yahoo.search.federation.ProviderConfig.Yca; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author gjoranv + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class HttpParametersTest { + + @Test + public void create_from_config() throws Exception { + ProviderConfig config = new ProviderConfig(new ProviderConfig.Builder() + .connectionTimeout(1.0) + .maxConnectionPerRoute(2) + .maxConnections(3) + .path("myPath") + .readTimeout(4) + .socketBufferBytes(5) + .yca(new Yca.Builder() + .applicationId("myId") + .host("myYcaHost") + .port(7) + .retry(8) + .ttl(9) + .useProxy(true))); + + HTTPParameters httpParameters = new HTTPParameters(config); + + // Written to configuredConnectionTimeout, but it is not accessible!? + //assertThat(httpParameters.getConnectionTimeout(), is(1000)); + + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getMaxConnectionsPerRoute(), is(2)); + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getMaxTotalConnections(), is(3)); + + assertThat(httpParameters.getPath(), is("/myPath")); + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getReadTimeout(), is(4)); + + // This value is not set from config by the constructor!? + //assertThat(httpParameters.getSocketBufferSizeBytes(), is(5)); + + + assertThat(httpParameters.getYcaUseProxy(), is(true)); + assertThat(httpParameters.getYcaApplicationId(), is("myId")); + assertThat(httpParameters.getYcaProxy(), is("myYcaHost")); + assertThat(httpParameters.getYcaPort(), is(7)); + assertThat(httpParameters.getYcaRetry(), is(8000L)); + assertThat(httpParameters.getYcaTtl(), is(9000L)); + } + + @Test + public void requireFreezeWorksForAccessors() { + HTTPParameters p = new HTTPParameters(); + boolean caught = false; + final int expected = 37; + p.setConnectionTimeout(expected); + assertEquals(expected, p.getConnectionTimeout()); + p.freeze(); + try { + p.setConnectionTimeout(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setReadTimeout(expected); + assertEquals(expected, p.getReadTimeout()); + p.freeze(); + try { + p.setReadTimeout(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setPersistentConnections(true); + assertTrue(p.getPersistentConnections()); + p.freeze(); + try { + p.setPersistentConnections(false); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + assertEquals("http", p.getProxyType()); + + p = new HTTPParameters(); + caught = false; + p.setEnableProxy(true); + assertTrue(p.getEnableProxy()); + p.freeze(); + try { + p.setEnableProxy(false); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setProxyHost("nalle"); + assertEquals("nalle", p.getProxyHost()); + p.freeze(); + try { + p.setProxyHost("jappe"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setProxyPort(expected); + assertEquals(expected, p.getProxyPort()); + p.freeze(); + try { + p.setProxyPort(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setMethod("POST"); + assertEquals("POST", p.getMethod()); + p.freeze(); + try { + p.setMethod("GET"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setSchema("gopher"); + assertEquals("gopher", p.getSchema()); + p.freeze(); + try { + p.setSchema("http"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setInputEncoding("iso-8859-15"); + assertEquals("iso-8859-15", p.getInputEncoding()); + p.freeze(); + try { + p.setInputEncoding("shift-jis"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setOutputEncoding("iso-8859-15"); + assertEquals("iso-8859-15", p.getOutputEncoding()); + p.freeze(); + try { + p.setOutputEncoding("shift-jis"); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setMaxTotalConnections(expected); + assertEquals(expected, p.getMaxTotalConnections()); + p.freeze(); + try { + p.setMaxTotalConnections(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setMaxConnectionsPerRoute(expected); + assertEquals(expected, p.getMaxConnectionsPerRoute()); + p.freeze(); + try { + p.setMaxConnectionsPerRoute(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setSocketBufferSizeBytes(expected); + assertEquals(expected, p.getSocketBufferSizeBytes()); + p.freeze(); + try { + p.setSocketBufferSizeBytes(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + + p = new HTTPParameters(); + caught = false; + p.setRetries(expected); + assertEquals(expected, p.getRetries()); + p.freeze(); + try { + p.setRetries(0); + } catch (IllegalStateException e) { + caught = true; + } + assertTrue(caught); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java new file mode 100644 index 00000000000..8edc1ca8dd8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.StupidSingleThreadedHttpServer; +import com.yahoo.search.federation.ProviderConfig.PingOption; +import com.yahoo.search.federation.http.Connection; +import com.yahoo.search.federation.http.HTTPProviderSearcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.StringEntity; +import org.junit.Test; + +import java.io.InputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.core.StringContains.containsString; +import static org.junit.Assert.assertThat; + +/** + * See bug #3234696. + * + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class HttpPostTestCase { + + @Test + public void testPostingSearcher() throws Exception { + StupidSingleThreadedHttpServer server = new StupidSingleThreadedHttpServer(); + server.start(); + + TestPostSearcher searcher = new TestPostSearcher(new ComponentId("foo:1"), + Arrays.asList(new Connection("localhost", server.getServerPort())), + "/"); + Query q = new Query(""); + q.setTimeout(10000000L); + Execution e = new Execution(searcher, Execution.Context.createContextStub()); + + searcher.search(q, e); + + assertThat(server.getRequest(), containsString("My POST body")); + server.stop(); + } + + private static class TestPostSearcher extends HTTPProviderSearcher { + public TestPostSearcher(ComponentId id, List<Connection> connections, String path) { + super(id, connections, httpParameters(path), Statistics.nullImplementation); + } + + private static HTTPParameters httpParameters(String path) { + HTTPParameters httpParameters = new HTTPParameters(path); + httpParameters.setPingOption(PingOption.Enum.DISABLE); + return httpParameters; + } + + @Override + protected HttpUriRequest createRequest(String method, URI uri, HttpEntity entity) { + HttpPost request = new HttpPost(uri); + request.setEntity(entity); + return request; + } + + @Override + protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { + try { + return new StringEntity("My POST body"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + @Override + public Map<String, String> getCacheKey(Query q) { + return new HashMap<>(0); + } + + @Override + public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { + // do nothing with the result + } + + @Override + protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { + //Empty + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java new file mode 100644 index 00000000000..c59dffb9cb7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.StupidSingleThreadedHttpServer; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import com.yahoo.text.Utf8; + +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Rudimentary http searcher test. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class HttpTestCase extends junit.framework.TestCase { + + private StupidSingleThreadedHttpServer httpServer; + private TestHTTPClientSearcher searcher; + + public void testSearcher() throws JAXBException { + Result result = searchUsingLocalhost(); + + assertEquals("ok", result.getQuery().properties().get("gotResponse")); + assertEquals(0, result.getQuery().errors().size()); + } + + private Result searchUsingLocalhost() { + searcher = new TestHTTPClientSearcher("test","localhost",getPort()); + Query query = new Query("/?query=test"); + + query.setWindow(0,10); + return searcher.search(query, new Execution(searcher, Execution.Context.createContextStub())); + } + + public void test_that_ip_address_set_on_meta_hit() { + Result result = searchUsingLocalhost(); + Hit metaHit = getFirstMetaHit(result.hits()); + String ip = (String) metaHit.getField(HTTPSearcher.LOG_IP_ADDRESS); + + assertEquals(ip, "127.0.0.1"); + } + + private Hit getFirstMetaHit(HitGroup hits) { + for (Iterator<Hit> i = hits.unorderedDeepIterator(); i.hasNext();) { + Hit hit = i.next(); + if (hit.isMeta()) + return hit; + } + return null; + } + + @Override + public void setUp() throws Exception { + httpServer = new StupidSingleThreadedHttpServer(0, 0) { + @Override + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 5\r\n" + + "\r\n" + + "hello"); + } + }; + httpServer.start(); + } + + private int getPort() { + return httpServer.getServerPort(); + } + + @Override + public void tearDown() throws Exception { + httpServer.stop(); + if (searcher != null) { + searcher.shutdownConnectionManagers(); + } + } + + private static class TestHTTPClientSearcher extends HTTPClientSearcher { + + public TestHTTPClientSearcher(String id, String hostName, int port) { + super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); + } + + private static List<Connection> toConnections(String hostName,int port) { + List<Connection> connections=new ArrayList<>(); + connections.add(new Connection(hostName,port)); + return connections; + } + + @Override + public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { + query.properties().set("gotResponse","ok"); + return query; + } + + @Override + public Map<String, String> getCacheKey(Query q) { + return null; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java new file mode 100644 index 00000000000..34791168db4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java @@ -0,0 +1,278 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.Ping; +import com.yahoo.prelude.Pong; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.StupidSingleThreadedHttpServer; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import com.yahoo.text.Utf8; +import com.yahoo.yolean.Exceptions; +import org.apache.http.HttpEntity; + +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Check for different keep-alive scenarios. What we really want to test + * is the server does not hang. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class PingTestCase extends junit.framework.TestCase { + static final int TIMEOUT_MS = 60000; + public void testNiceCase() throws Exception { + NiceStupidServer server = new NiceStupidServer(); + server.start(); + checkSearchAndPing(true, true, true, server.getServerPort()); + server.stop(); + } + + private void checkSearchAndPing(boolean firstSearch, boolean pongCheck, boolean secondSearch, int port) { + String resultThing; + String comment; + TestHTTPClientSearcher searcher = new TestHTTPClientSearcher("test", + "localhost", port); + try { + + Query query = new Query("/?query=test"); + + query.setWindow(0, 10); + // high timeout to allow for overloaded test machine + query.setTimeout(TIMEOUT_MS); + Ping ping = new Ping(TIMEOUT_MS); + + long start = System.currentTimeMillis(); + Execution exe = new Execution(searcher, Execution.Context.createContextStub()); + exe.search(query); + + resultThing = firstSearch ? "ok" : null; + comment = firstSearch ? "First search should have succeeded." : "First search should fail."; + assertEquals(comment, resultThing, query.properties().get("gotResponse")); + Pong pong = searcher.ping(ping, searcher.getConnection()); + if (pongCheck) { + assertEquals("Ping should not have failed.", 0, pong.getErrorSize()); + } else { + assertEquals("Ping should have failed.", 1, pong.getErrorSize()); + } + exe = new Execution(searcher, Execution.Context.createContextStub()); + exe.search(query); + + resultThing = secondSearch ? "ok" : null; + comment = secondSearch ? "Second search should have succeeded." : "Second search should fail."; + + assertEquals(resultThing, query.properties().get("gotResponse")); + long duration = System.currentTimeMillis() - start; + // target for duration based on the timeout values + some slack + assertTrue("This test probably hanged.", duration < TIMEOUT_MS + 4000); + searcher.shutdownConnectionManagers(); + } finally { + searcher.deconstruct(); + } + } + + public void testUselessCase() throws Exception { + UselessStupidServer server = new UselessStupidServer(); + server.start(); + checkSearchAndPing(false, true, false, server.getServerPort()); + server.stop(); + } + + public void testGrumpyCase() throws Exception { + GrumpyStupidServer server = new GrumpyStupidServer(); + server.start(); + checkSearchAndPing(false, false, false, server.getServerPort()); + server.stop(); + } + + public void testPassiveAggressiveCase() throws Exception { + PassiveAggressiveStupidServer server = new PassiveAggressiveStupidServer(); + server.start(); + checkSearchAndPing(true, false, true, server.getServerPort()); + server.stop(); + } + + // OK on ping and search + private static class NiceStupidServer extends StupidSingleThreadedHttpServer { + private NiceStupidServer() throws IOException { + super(0, 0); + } + + @Override + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "hello\n"); + } + } + + // rejects ping and accepts search + private static class PassiveAggressiveStupidServer extends StupidSingleThreadedHttpServer { + + private PassiveAggressiveStupidServer() throws IOException { + super(0, 0); + } + + @Override + protected byte[] getResponse(String request) { + if (request.contains("/ping")) { + return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 8\r\n" + + "\r\n" + + "go away\n"); + } else { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "hello\n"); + } + } + } + + // accepts ping and rejects search + private static class UselessStupidServer extends StupidSingleThreadedHttpServer { + private UselessStupidServer() throws IOException { + super(0, 0); + } + + + @Override + protected byte[] getResponse(String request) { + if (request.contains("/ping")) { + return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "hello\n"); + } else { + return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 8\r\n" + + "\r\n" + + "go away\n"); + } + } + } + + // rejects ping and search + private static class GrumpyStupidServer extends StupidSingleThreadedHttpServer { + private GrumpyStupidServer() throws IOException { + super(0, 0); + } + + @Override + protected byte[] getResponse(String request) { + return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + + "Content-Type: text/xml; charset=UTF-8\r\n" + + "Connection: close\r\n" + + "Content-Length: 8\r\n" + + "\r\n" + + "go away\n"); + } + } + + private static class TestHTTPClientSearcher extends HTTPClientSearcher { + + public TestHTTPClientSearcher(String id, String hostName, int port) { + super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); + } + + private static List<Connection> toConnections(String hostName,int port) { + List<Connection> connections=new ArrayList<>(); + connections.add(new Connection(hostName,port)); + return connections; + } + + @Override + public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { + query.properties().set("gotResponse","ok"); + return query; + } + + @Override + public Result search(Query query, Execution execution, + Connection connection) { + URI uri; + try { + uri = new URL("http", connection.getHost(), connection + .getPort(), "/search").toURI(); + } catch (MalformedURLException e) { + query.errors().add(createMalformedUrlError(query, e)); + return execution.search(query); + } catch (URISyntaxException e) { + query.errors().add(createMalformedUrlError(query, e)); + return execution.search(query); + } + + HttpEntity entity; + try { + entity = getEntity(uri, query); + } catch (IOException e) { + query.errors().add( + ErrorMessage.createBackendCommunicationError("Error when trying to connect to HTTP backend in " + + this + " using " + connection + + " for " + query + ": " + + Exceptions.toMessageString(e))); + return execution.search(query); + } catch (TimeoutException e) { + query.errors().add(ErrorMessage.createTimeout("No time left for HTTP traffic in " + + this + + " for " + query + ": " + e.getMessage())); + return execution.search(query); + } + if (entity == null) { + query.errors().add( + ErrorMessage.createBackendCommunicationError("No result from connecting to HTTP backend in " + + this + " using " + connection + " for " + query)); + return execution.search(query); + } + + try { + query = handleResponse(entity, query); + } catch (IOException e) { + query.errors().add( + ErrorMessage.createBackendCommunicationError("Error when trying to consume input in " + + this + ": " + Exceptions.toMessageString(e))); + } finally { + cleanupHttpEntity(entity); + } + return execution.search(query); + } + + @Override + public Map<String, String> getCacheKey(Query q) { + return null; + } + + @Override + protected URI getPingURI(Connection connection) + throws MalformedURLException, URISyntaxException { + return new URL("http", connection.getHost(), connection.getPort(), "/ping").toURI(); + } + + Connection getConnection() { + return getHasher().getNodes().select(0, 0); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java new file mode 100644 index 00000000000..baeb9fd0a41 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.federation.vespa.VespaSearcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.statistics.Statistics; +import com.yahoo.vespa.defaults.Defaults; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Tests that source and backend specific parameters from the query are added correctly to the backend requests + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class QueryParametersTestCase extends junit.framework.TestCase { + + public void testQueryParameters() { + Query query=new Query(); + query.properties().set("a","a-value"); + query.properties().set("b.c","b.c-value"); + query.properties().set("source.otherSource.d","d-value"); + query.properties().set("source.testSource.e","e-value"); + query.properties().set("source.testSource.f.g","f.g-value"); + query.properties().set("provider.testProvider.h","h-value"); + query.properties().set("provider.testProvider.i.j","i.j-value"); + + query.properties().set("sourceName","testSource"); // Done by federation searcher + query.properties().set("providerName","testProvider"); // Done by federation searcher + + TestHttpProvider searcher=new TestHttpProvider(); + Map<String,String> parameters=searcher.getQueryMap(query); + searcher.deconstruct(); + + assertEquals(4,parameters.size()); // the appropriate 4 of the above + assertEquals(parameters.get("e"),"e-value"); + assertEquals(parameters.get("f.g"),"f.g-value"); + assertEquals(parameters.get("h"),"h-value"); + assertEquals(parameters.get("i.j"),"i.j-value"); + } + + public static class TestHttpProvider extends HTTPProviderSearcher { + + public TestHttpProvider() { + super(new ComponentId("test"), Collections.singletonList(new Connection("host", Defaults.getDefaults().vespaWebServicePort())), "path", Statistics.nullImplementation); + } + + @Override + public Map<String, String> getCacheKey(Query q) { + return Collections.singletonMap("nocaching", String.valueOf(Math.random())); + } + + @Override + protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { + } + + } + +} + diff --git a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java new file mode 100644 index 00000000000..e874c89b918 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SearchChainResolverTestCase.java @@ -0,0 +1,152 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.processing.request.properties.PropertyMap; +import com.yahoo.processing.request.Properties; +import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.federation.sourceref.Target; +import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.junit.Test; + +import java.util.Collections; +import java.util.Iterator; +import java.util.SortedSet; + +import static junit.framework.Assert.assertNull; +import static junit.framework.Assert.fail; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class SearchChainResolverTestCase { + + private static final FederationOptions federationOptions = + new FederationOptions().setTimeoutInMilliseconds(3000).setOptional(true); + + private static final ComponentId searchChainId = ComponentId.fromString("search-chain"); + private static final ComponentId providerId = ComponentId.fromString("provider"); + private static final ComponentId provider2Id = ComponentId.fromString("provider2"); + + private static final ComponentId sourceId = ComponentId.fromString("source"); + private static final ComponentId sourceChainInProviderId = + ComponentId.fromString("source-chain").nestInNamespace(providerId); + private static final ComponentId sourceChainInProvider2Id = + ComponentId.fromString("source-chain").nestInNamespace(provider2Id); + + private static final SearchChainResolver searchChainResolver; + + static { + SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + builder.addSearchChain(searchChainId, federationOptions.setUseByDefault(true), Collections.<String>emptyList()); + builder.addSearchChain(providerId, federationOptions.setUseByDefault(false), Collections.<String>emptyList()); + builder.addSourceForProvider(sourceId, providerId, sourceChainInProviderId, true, + federationOptions.setUseByDefault(true), Collections.<String>emptyList()); + builder.addSourceForProvider(sourceId, provider2Id, sourceChainInProvider2Id, false, + federationOptions.setUseByDefault(false), Collections.<String>emptyList()); + + searchChainResolver = builder.build(); + } + + @Test + public void check_default_search_chains() { + assertThat(searchChainResolver.defaultTargets().size(), is(2)); + + Iterator<Target> iterator = searchChainResolver.defaultTargets().iterator(); + assertThat(iterator.next().searchRefDescription(), is(searchChainId.toString())); + assertThat(iterator.next().searchRefDescription(), is(sourceChainInProviderId.toString())); + } + + @Test + public void require_error_message_for_invalid_source() { + try { + resolve("no-such-source"); + fail("Expected exception."); + } catch (UnresolvedSearchChainException e) { + assertThat(e.getMessage(), is("Could not resolve source ref 'no-such-source'.")); + } + } + + @Test + public void lookup_search_chain() throws Exception { + SearchChainInvocationSpec res = resolve(searchChainId.getName()); + assertThat(res.searchChainId, is(searchChainId)); + } + + //TODO: TVT: @Test() + public void lookup_provider() throws Exception { + SearchChainInvocationSpec res = resolve(providerId.getName()); + assertThat(res.provider, is(providerId)); + assertNull(res.source); + assertThat(res.searchChainId, is(providerId)); + } + + @Test + public void lookup_source() throws Exception { + SearchChainInvocationSpec res = resolve(sourceId.getName()); + assertIsSourceInProvider(res); + } + + @Test + public void lookup_source_search_chain_directly() throws Exception { + SearchChainInvocationSpec res = resolve(sourceChainInProviderId.stringValue()); + assertIsSourceInProvider(res); + } + + private void assertIsSourceInProvider(SearchChainInvocationSpec res) { + assertThat(res.provider, is(providerId)); + assertThat(res.source, is(sourceId)); + assertThat(res.searchChainId, is(sourceChainInProviderId)); + } + + @Test + public void lookup_source_for_provider2() throws Exception { + SearchChainInvocationSpec res = resolve(sourceId.getName(), provider2Id.getName()); + assertThat(res.provider, is(provider2Id)); + assertThat(res.source, is(sourceId)); + assertThat(res.searchChainId, is(sourceChainInProvider2Id)); + } + + @Test + public void lists_source_ref_description_for_top_level_targets() { + SortedSet<Target> topLevelTargets = searchChainResolver.allTopLevelTargets(); + assertThat(topLevelTargets.size(), is(3)); + + Iterator<Target> i = topLevelTargets.iterator(); + assertSearchRefDescriptionIs(i.next(), providerId.toString()); + assertSearchRefDescriptionIs(i.next(), searchChainId.toString()); + assertSearchRefDescriptionIs(i.next(), "source[provider = provider, provider2]"); + } + + private void assertSearchRefDescriptionIs(Target target, String expected) { + assertThat(target.searchRefDescription(), is(expected)); + } + + static Properties emptySourceToProviderMap() { + return new PropertyMap(); + } + + private SearchChainInvocationSpec resolve(String sourceSpecification) throws UnresolvedSearchChainException { + return resolve(sourceSpecification, emptySourceToProviderMap()); + } + + private SearchChainInvocationSpec resolve(String sourceSpecification, String providerSpecification) + throws UnresolvedSearchChainException { + Properties sourceToProviderMap = emptySourceToProviderMap(); + sourceToProviderMap.set("source." + sourceSpecification + ".provider", providerSpecification); + return resolve(sourceSpecification, sourceToProviderMap); + } + + private SearchChainInvocationSpec resolve(String sourceSpecification, Properties sourceToProviderMap) + throws UnresolvedSearchChainException { + SearchChainInvocationSpec res = searchChainResolver.resolve( + ComponentSpecification.fromString(sourceSpecification), sourceToProviderMap); + assertThat(res.federationOptions, is(federationOptions)); + return res; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java new file mode 100644 index 00000000000..f8559745358 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/sourceref/test/SourceRefResolverTestCase.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.sourceref.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.search.federation.sourceref.SearchChainInvocationSpec; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.federation.sourceref.SourceRefResolver; +import com.yahoo.search.federation.sourceref.UnresolvedSearchChainException; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeMap; + +import static com.yahoo.search.federation.sourceref.test.SearchChainResolverTestCase.emptySourceToProviderMap; +import static junit.framework.Assert.fail; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.matchers.JUnitMatchers.hasItems; + + +/** + * Test for SourceRefResolver. + * @author tonytv + */ +public class SourceRefResolverTestCase { + private static final String cluster1 = "cluster1"; + private static final String cluster2 = "cluster2"; + private static final String cluster3 = "cluster3"; + private static IndexFacts indexFacts; + + private static final SourceRefResolver sourceRefResolver = createSourceRefResolver(); + + static { + setupIndexFacts(); + } + + private static SourceRefResolver createSourceRefResolver() { + SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + builder.addSearchChain(ComponentId.fromString(cluster1), new FederationOptions().setUseByDefault(true), + Collections.<String>emptyList()); + builder.addSearchChain(ComponentId.fromString(cluster2), new FederationOptions().setUseByDefault(true), + Collections.<String>emptyList()); + + return new SourceRefResolver(builder.build()); + } + + private static void setupIndexFacts() { + TreeMap<String, List<String>> masterClusters = new TreeMap<>(); + masterClusters.put(cluster1, Arrays.asList("document1", "document2")); + masterClusters.put(cluster2, Arrays.asList("document1")); + masterClusters.put(cluster3, Arrays.asList("document3")); + indexFacts = new IndexFacts(new IndexModel(masterClusters, null, null)); + } + + @Test + public void check_test_assumptions() { + assertThat(indexFacts.clustersHavingSearchDefinition("document1"), hasItems("cluster1", "cluster2")); + } + + @Test + public void lookup_search_chain() throws Exception { + Set<SearchChainInvocationSpec> searchChains = resolve(cluster1); + assertThat(searchChains.size(), is(1)); + assertThat(searchChainIds(searchChains), hasItems(cluster1)); + } + + @Test + public void lookup_search_chains_for_document1() throws Exception { + Set<SearchChainInvocationSpec> searchChains = resolve("document1"); + assertThat(searchChains.size(), is(2)); + assertThat(searchChainIds(searchChains), hasItems(cluster1, cluster2)); + } + + @Test + public void error_when_document_gives_cluster_without_matching_search_chain() { + try { + resolve("document3"); + fail("Expected exception"); + } catch (UnresolvedSearchChainException e) { + assertThat(e.getMessage(), is("Failed to resolve cluster search chain 'cluster3' " + + "when using source ref 'document3' as a document name.")); + } + } + + @Test + public void error_when_no_document_or_search_chain() { + try { + resolve("document4"); + fail("Expected exception"); + } catch (UnresolvedSearchChainException e) { + assertThat(e.getMessage(), is("Could not resolve source ref 'document4'.")); + } + } + + private List<String> searchChainIds(Set<SearchChainInvocationSpec> searchChains) { + List<String> names = new ArrayList<>(); + for (SearchChainInvocationSpec searchChain : searchChains) { + names.add(searchChain.searchChainId.stringValue()); + } + return names; + } + + private Set<SearchChainInvocationSpec> resolve(String documentName) throws UnresolvedSearchChainException { + return sourceRefResolver.resolve(ComponentSpecification.fromString(documentName), emptySourceToProviderMap(), indexFacts); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java b/container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java new file mode 100644 index 00000000000..40786ee89a9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/AddHitsWithRelevanceSearcher.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author tonytv + */ +public class AddHitsWithRelevanceSearcher extends Searcher { + public static final int numHitsAdded = 5; + + private final String chainName; + private final int relevanceMultiplier; + + public AddHitsWithRelevanceSearcher(String chainName, int rankMultiplier) { + this.chainName = chainName; + this.relevanceMultiplier = rankMultiplier; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + for (int i = 1; i <= numHitsAdded; ++i) { + result.hits().add(createHit(i)); + } + return result; + } + + private Hit createHit(int i) { + int relevance = i * relevanceMultiplier; + return new Hit(chainName + "-" + relevance, relevance); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java b/container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java new file mode 100644 index 00000000000..dcecf36f2ae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/BlockingSearcher.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author tonytv + */ +public class BlockingSearcher extends Searcher { + @Override + public synchronized Result search(Query query, Execution execution) { + try { + while (true) + wait(); + } catch (InterruptedException e) { + } + return execution.search(query); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java new file mode 100644 index 00000000000..dba0deb607a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTest.java @@ -0,0 +1,306 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import java.util.Optional; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.net.URI; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.processing.execution.chain.ChainRegistry; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.selection.FederationTarget; +import com.yahoo.search.federation.selection.TargetSelector; +import com.yahoo.search.federation.StrictContractsConfig; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.Execution.Context; +import com.yahoo.search.searchchain.model.federation.FederationOptions; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; + +/** + * @author tonytv + */ +public class FederationSearcherTest { + private static final String hasBeenFilled = "hasBeenFilled"; + + private static class AddHitSearcher extends Searcher { + protected Hit hit = createHit(); + + private Hit createHit() { + Hit hit = new Hit("dummy"); + hit.setFillable(); + return hit; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(hit); + return result; + } + + @Override + public void fill(Result result, String summaryClass, Execution execution) { + if (firstHit(result) != hit) { + throw new RuntimeException("Unknown hit"); + } + firstHit(result).setField(hasBeenFilled, true); + } + } + + private static class ModifyQueryAndAddHitSearcher extends AddHitSearcher { + private final String marker; + + ModifyQueryAndAddHitSearcher(String marker) { + super(); + this.marker = marker; + } + + @Override + public Result search(Query query, Execution execution) { + query.getModel().getQueryTree().setRoot(new WordItem(marker)); + Result result = execution.search(query); + result.hits().add(hit); + return result; + } + + } + + @Test + public void require_that_hits_are_not_automatically_filled() { + Result result = federationToSingleAddHitSearcher().search(); + assertNotFilled(firstHitInFirstGroup(result)); + } + + @Test + public void require_that_hits_can_be_filled() { + Result result = federationToSingleAddHitSearcher().searchAndFill(); + assertFilled(firstHitInFirstGroup(result)); + } + + @Test + public void require_that_hits_can_be_filled_when_moved() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + tester.addSearchChain("chain2", new AddHitSearcher()); + + Result result = tester.search(); + + Result reorganizedResult = new Result(result.getQuery()); + HitGroup hit1 = new HitGroup(); + HitGroup nestedHitGroup = new HitGroup(); + + hit1.add(nestedHitGroup); + reorganizedResult.hits().add(hit1); + + HitGroup chain1Group = (HitGroup) result.hits().get(0); + HitGroup chain2Group = (HitGroup) result.hits().get(1); + + nestedHitGroup.add(chain1Group.get(0)); + reorganizedResult.hits().add(chain2Group.get(0)); + reorganizedResult.hits().add(nestedHitGroup); + + tester.fill(reorganizedResult); + assertFilled(nestedHitGroup.get(0)); + assertFilled(chain2Group.get(0)); + + } + + @Test + public void require_that_hits_can_be_filled_for_multiple_chains_and_queries() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + tester.addSearchChain("chain2", new ModifyQueryAndAddHitSearcher("modified1")); + tester.addSearchChain("chain3", new ModifyQueryAndAddHitSearcher("modified2")); + + Result result = tester.search(); + tester.fill(result); + for (Iterator<Hit> i = result.hits().deepIterator(); i.hasNext();) { + Hit h = i.next(); + assertFilled(h); + } + assertEquals(3, result.hits().getConcreteSize()); + } + + + @Test + public void require_that_optional_search_chains_does_not_delay_federation() { + BlockingSearcher blockingSearcher = new BlockingSearcher(); + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + tester.addOptionalSearchChain("chain2", blockingSearcher); + + Result result = tester.searchAndFill(); + assertThat(getNonErrorHits(result).size(), is(1)); + assertFilled(getFirstHit(getNonErrorHits(result).get(0))); + assertNotNull(result.hits().getError()); + } + + @Test + public void require_that_calling_a_single_slow_source_with_long_timeout_does_not_delay_federation() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", + new FederationOptions().setUseByDefault(true).setRequestTimeoutInMilliseconds(3600 * 1000), + new BlockingSearcher() ); + + Query query = new Query(); + query.setTimeout(2); // make the test run faster + Result result = tester.search(query); + assertThat(getNonErrorHits(result).size(), is(0)); + assertNotNull(result.hits().getError()); + } + + private Hit getFirstHit(Hit hitGroup) { + if (hitGroup instanceof HitGroup) + return ((HitGroup) hitGroup).get(0); + else + throw new IllegalArgumentException("Expected HitGroup"); + } + + private List<Hit> getNonErrorHits(Result result) { + List<Hit> nonErrorHits = new ArrayList<>(); + for (Hit hit : result.hits()) { + if (!(hit instanceof ErrorHit)) + nonErrorHits.add(hit); + } + + return nonErrorHits; + } + private static void assertFilled(Hit hit) { + assertTrue((Boolean)hit.getField(hasBeenFilled)); + } + + private static void assertNotFilled(Hit hit) { + assertNull(hit.getField(hasBeenFilled)); + } + + private FederationTester federationToSingleAddHitSearcher() { + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitSearcher()); + return tester; + } + + private static Hit firstHit(Result result) { + return result.hits().get(0); + } + + private static Hit firstHitInFirstGroup(Result result) { + return ((HitGroup)firstHit(result)).get(0); + } + + @Test + public void custom_federation_target() { + ComponentId targetSelectorId = ComponentId.fromString("TargetSelector"); + ComponentRegistry<TargetSelector> targetSelectors = new ComponentRegistry<>(); + targetSelectors.register(targetSelectorId, new TestTargetSelector()); + + FederationSearcher searcher = new FederationSearcher( + new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), + new StrictContractsConfig(new StrictContractsConfig.Builder()), + targetSelectors); + + Result result = new Execution(searcher, Context.createContextStub()).search(new Query()); + HitGroup myChainGroup = (HitGroup) result.hits().get(0); + assertThat(myChainGroup.getId(), is(new URI("source:myChain"))); + assertThat(myChainGroup.get(0).getId(), is(new URI("myHit"))); + } + + static class TestTargetSelector implements TargetSelector<String> { + String keyName = getClass().getName(); + + @Override + public Collection<FederationTarget<String>> getTargets(Query query, ChainRegistry<Searcher> searcherChainRegistry) { + return Arrays.asList( + new FederationTarget<>(new Chain<>("myChain", Collections.<Searcher>emptyList()), new FederationOptions(), "hello")); + } + + @Override + public void modifyTargetQuery(FederationTarget<String> target, Query query) { + checkTarget(target); + query.properties().set(keyName, "called"); + } + + @Override + public void modifyTargetResult(FederationTarget<String> target, Result result) { + checkTarget(target); + assertThat(result.getQuery().properties().getString(keyName), is("called")); + result.hits().add(new Hit("myHit")); + } + + private void checkTarget(FederationTarget<String> target) { + assertThat(target.getCustomData(), is("hello")); + assertThat(target.getChain().getId(), is(ComponentId.fromString("myChain"))); + } + } + + static class TestMultipleTargetSelector implements TargetSelector<String> { + String keyName = getClass().getName(); + + @Override + public Collection<FederationTarget<String>> getTargets(Query query, ChainRegistry<Searcher> searcherChainRegistry) { + return Arrays.asList(createTarget(1), createTarget(2)); + } + + private FederationTarget<String> createTarget(int number) { + return new FederationTarget<>(new Chain<>("chain" + number, Collections.<Searcher>emptyList()), + new FederationOptions(), + "custom-data:" + number); + } + + @Override + public void modifyTargetQuery(FederationTarget<String> target, Query query) { + query.properties().set(keyName, "modifyTargetQuery:" + target.getCustomData()); + } + + @Override + public void modifyTargetResult(FederationTarget<String> target, Result result) { + Hit hit = new Hit("MyHit" + target.getCustomData()); + hit.setField("data", result.getQuery().properties().get(keyName)); + result.hits().add(hit); + } + } + + @Test + public void target_selectors_can_have_multiple_targets() { + ComponentId targetSelectorId = ComponentId.fromString("TestMultipleTargetSelector"); + ComponentRegistry<TargetSelector> targetSelectors = new ComponentRegistry<>(); + targetSelectors.register(targetSelectorId, new TestMultipleTargetSelector()); + + FederationSearcher searcher = new FederationSearcher( + new FederationConfig(new FederationConfig.Builder().targetSelector(targetSelectorId.toString())), + new StrictContractsConfig(new StrictContractsConfig.Builder()), + targetSelectors); + + Result result = new Execution(searcher, Context.createContextStub()).search(new Query()); + + Iterator<Hit> hitsIterator = result.hits().deepIterator(); + Hit hit1 = hitsIterator.next(); + Hit hit2 = hitsIterator.next(); + + assertThat(hit1.getSource(), is("chain1")); + assertThat(hit2.getSource(), is("chain2")); + + assertThat((String)hit1.getField("data"), is("modifyTargetQuery:custom-data:1")); + assertThat((String)hit2.getField("data"), is("modifyTargetQuery:custom-data:2")); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java new file mode 100644 index 00000000000..bc00890624b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationSearcherTestCase.java @@ -0,0 +1,411 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.StrictContractsConfig; +import com.yahoo.search.federation.selection.TargetSelector; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.test.QueryTestCase; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; + +import static org.junit.Assert.*; +import static com.yahoo.search.federation.StrictContractsConfig.PropagateSourceProperties; + +/** + * Test for federation searcher. The searcher is also tested in + * com.yahoo.prelude.searcher.test.BlendingSearcherTestCase. + * + * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + */ +@SuppressWarnings("deprecation") +public class FederationSearcherTestCase { + + static final String SOURCE1 = "source1"; + static final String SOURCE2 = "source2"; + + public static class TwoSourceChecker extends TraceVisitor { + public boolean traceFromSource1 = false; + public boolean traceFromSource2 = false; + + @Override + public void visit(TraceNode node) { + if (SOURCE1.equals(node.payload())) { + traceFromSource1 = true; + } else if (SOURCE2.equals(node.payload())) { + traceFromSource2 = true; + } + } + + } + + private FederationConfig.Builder builder; + private SearchChainRegistry chainRegistry; + + @Before + public void setUp() throws Exception { + builder = new FederationConfig.Builder(); + chainRegistry = new SearchChainRegistry(); + } + + @After + public void tearDown() throws Exception { + builder = null; + chainRegistry = null; + } + + private void addChained(final Searcher searcher, final String sourceName) { + builder.target(new FederationConfig.Target.Builder(). + id(sourceName). + searchChain(new FederationConfig.Target.SearchChain.Builder(). + searchChainId(sourceName). + timeoutMillis(10000). + useByDefault(true)) + ); + chainRegistry.register(new ComponentId(sourceName), + createSearchChain(new ComponentId(sourceName), searcher)); + } + + private Searcher createFederationSearcher() { + return buildFederation(new StrictContractsConfig(new StrictContractsConfig.Builder())); + } + + private Searcher createFederationSearcher(PropagateSourceProperties.Enum propagateSourceProperties) { + return buildFederation(new StrictContractsConfig(new StrictContractsConfig.Builder().propagateSourceProperties(propagateSourceProperties))); + } + + private Searcher createStrictFederationSearcher() { + StrictContractsConfig.Builder builder = new StrictContractsConfig.Builder(); + builder.searchchains(true); + final StrictContractsConfig contracts = new StrictContractsConfig(builder); + return buildFederation(contracts); + } + + private Searcher buildFederation(final StrictContractsConfig contracts) + throws RuntimeException { + + return new FederationSearcher(new FederationConfig(builder), contracts, new ComponentRegistry<TargetSelector>()); + } + + private SearchChain createSearchChain(final ComponentId chainId, + final Searcher searcher) { + return new SearchChain(chainId, searcher); + } + + @Test + public void testQueryProfileNestedReferencing() { + addChained(new MockSearcher(), "mySource1"); + addChained(new MockSearcher(), "mySource2"); + Chain<Searcher> mainChain = new Chain<>("default", createFederationSearcher()); + + QueryProfile defaultProfile = new QueryProfile("default"); + defaultProfile.set("source.mySource1.hits", "%{hits}", (QueryProfileRegistry)null); + defaultProfile.freeze(); + Query q = new Query(QueryTestCase.httpEncode("?query=test"), defaultProfile.compile(null)); + + Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); + assertNull(result.hits().getError()); + assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); + } + + @Test + public void testTraceTwoSources() { + final Chain<Searcher> mainChain = twoTracingSources(false); + + final Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1")); + + final Execution execution = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)); + final Result result = execution.search(q); + assertNull(result.hits().getError()); + TwoSourceChecker lookForTraces = new TwoSourceChecker(); + execution.trace().accept(lookForTraces); + assertTrue(lookForTraces.traceFromSource1); + assertTrue(lookForTraces.traceFromSource2); + } + + private Chain<Searcher> twoTracingSources(boolean strictContracts) { + addChained(new Searcher() { + @Override + public Result search(Query query, Execution execution) { + query.trace(SOURCE1, 1); + return execution.search(query); + } + + }, SOURCE1); + + addChained(new Searcher() { + @Override + public Result search(Query query, Execution execution) { + query.trace(SOURCE2, 1); + return execution.search(query); + } + + }, SOURCE2); + + final Chain<Searcher> mainChain = new Chain<>("default", + new FederationSearcher(new FederationConfig(builder), + new StrictContractsConfig( + new StrictContractsConfig.Builder().searchchains(strictContracts)), + new ComponentRegistry<>())); + return mainChain; + } + + @Test + public void testTraceOneSourceNoCloning() { + final Chain<Searcher> mainChain = twoTracingSources(true); + + final Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1&sources=source1")); + + final Execution execution = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)); + final Result result = execution.search(q); + assertNull(result.hits().getError()); + TwoSourceChecker lookForTraces = new TwoSourceChecker(); + execution.trace().accept(lookForTraces); + assertTrue(lookForTraces.traceFromSource1); + assertFalse(lookForTraces.traceFromSource2); + } + + @Test + public void testTraceOneSourceWithCloning() { + final Chain<Searcher> mainChain = twoTracingSources(false); + + final Query q = new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&traceLevel=1&sources=source1")); + + final Execution execution = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)); + final Result result = execution.search(q); + assertNull(result.hits().getError()); + TwoSourceChecker lookForTraces = new TwoSourceChecker(); + execution.trace().accept(lookForTraces); + assertTrue(lookForTraces.traceFromSource1); + assertFalse(lookForTraces.traceFromSource2); + + } + + + @Test + public void testPropertyPropagation() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.ALL); + + assertEquals("source:mySource1", result.hits().get(0).getId() + .stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId() + .stringValue()); + assertEquals("nalle", result.hits().get(0).getQuery().getPresentation() + .getSummary()); + assertNull(result.hits().get(1).getQuery().getPresentation() + .getSummary()); + + } + + private Result searchWithPropertyPropagation(PropagateSourceProperties.Enum propagateSourceProperties) { + addChained(new MockSearcher(), "mySource1"); + addChained(new MockSearcher(), "mySource2"); + final Chain<Searcher> mainChain = new Chain<>("default", createFederationSearcher(propagateSourceProperties)); + + final Query q = new Query(QueryTestCase.httpEncode("?query=test&source.mySource1.presentation.summary=nalle")); + + final Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); + assertNull(result.hits().getError()); + return result; + } + + @Test + public void testDisablePropertyPropagation() { + Result result = searchWithPropertyPropagation(PropagateSourceProperties.NONE); + + assertNull(result.hits().get(0).getQuery().getPresentation() + .getSummary()); + } + + @Test + public void testNoCloning() { + final String sourceName = "cloningcheck"; + Query query = new Query(QueryTestCase.httpEncode("?query=test&sources=" + sourceName)); + addChained(new QueryCheckSearcher(query), sourceName); + addChained(new MockSearcher(), "mySource1"); + Chain<Searcher> mainChain = new Chain<>("default", createStrictFederationSearcher()); + Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(query); + HitGroup h = (HitGroup) result.hits().get(0); + assertNull(h.getErrorHit()); + assertSame(QueryCheckSearcher.OK, h.get(0).getField(QueryCheckSearcher.STATUS)); + + mainChain = new Chain<>("default", createFederationSearcher()); + result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(query); + h = (HitGroup) result.hits().get(0); + assertSame(QueryCheckSearcher.FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY, + h.getError().getDetailedMessage()); + + query = new Query(QueryTestCase.httpEncode("?query=test&sources=" + sourceName + ",mySource1")); + addChained(new QueryCheckSearcher(query), sourceName); + result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(query); + h = (HitGroup) result.hits().get(0); + assertEquals("source:" + sourceName, h.getId().stringValue()); + assertSame(QueryCheckSearcher.FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY, + h.getError().getDetailedMessage()); + assertEquals("source:mySource1", result.hits().get(1).getId() + .stringValue()); + } + + @Test + public void testTopLevelHitGroupFieldPropagation() { + addChained(new MockSearcher(), "mySource1"); + addChained(new AnotherMockSearcher(), "mySource2"); + Chain<Searcher> mainChain = new Chain<>("default", createFederationSearcher()); + + Query q = new Query("?query=test"); + + Result result = new Execution(mainChain, Execution.Context.createContextStub(chainRegistry, null)).search(q); + assertNull(result.hits().getError()); + assertEquals("source:mySource1", result.hits().get(0).getId().stringValue()); + assertEquals("source:mySource2", result.hits().get(1).getId().stringValue()); + assertEquals( + AnotherMockSearcher.IS_THIS_PROPAGATED, + result.hits().get(1).getField(AnotherMockSearcher.PROPAGATION_KEY)); + } + + private static class MockSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + String sourceName = query.properties().getString("sourceName", "unknown"); + Result result = new Result(query); + for (int i = 1; i <= query.getHits(); i++) { + final Hit hit = new Hit(sourceName + ":" + i, 1d / i); + hit.setSource(sourceName); + result.hits().add(hit); + } + return result; + } + + } + + private static class SleepingMockSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + try { + Thread.sleep(100); + } catch (Exception e) { + throw new RuntimeException(e); + } + return execution.search(query); + } + } + + + private static class AnotherMockSearcher extends Searcher { + + private static final String PROPAGATION_KEY = "hello"; + private static final String IS_THIS_PROPAGATED = "is this propagated?"; + + @Override + public Result search(final Query query, final Execution execution) { + final Result result = new Result(query); + result.hits().setField(PROPAGATION_KEY, IS_THIS_PROPAGATED); + return result; + } + } + + @Test + public void testProviderSelectionFromQueryProperties() { + SearchChainRegistry registry = new SearchChainRegistry(); + registry.register(new Chain<>("provider1", new MockProvider("provider1"))); + registry.register(new Chain<>("provider2", new MockProvider("provider2"))); + registry.register(new Chain<>("default", createMultiProviderFederationSearcher())); + assertSelects("provider1", registry); + assertSelects("provider2", registry); + } + + private void assertSelects(String providerName, SearchChainRegistry registry) { + QueryProfile profile = new QueryProfile("test"); + profile.set("source.news.provider", providerName, (QueryProfileRegistry)null); + Query query = new Query(QueryTestCase.httpEncode("?query=test&model.sources=news"), profile.compile(null)); + Result result = new Execution(registry.getComponent("default"), Execution.Context.createContextStub(registry, null)).search(query); + assertEquals(1, result.hits().size()); + assertNotNull(result.hits().get(providerName + ":1")); + } + + private FederationSearcher createMultiProviderFederationSearcher() { + final FederationOptions options = new FederationOptions(); + final SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + + final ComponentId provider1 = new ComponentId("provider1"); + final ComponentId provider2 = new ComponentId("provider2"); + final ComponentId news = new ComponentId("news"); + builder.addSearchChain(provider1, options, + Collections.<String> emptyList()); + builder.addSearchChain(provider2, options, + Collections.<String> emptyList()); + builder.addSourceForProvider(news, provider1, provider1, true, options, + Collections.<String> emptyList()); + builder.addSourceForProvider(news, provider2, provider2, false, + options, Collections.<String> emptyList()); + + return new FederationSearcher(new ComponentId("federation"), builder.build()); + } + + private static class MockProvider extends Searcher { + + private final String name; + + public MockProvider(final String name) { + this.name = name; + } + + @Override + public Result search(final Query query, final Execution execution) { + final Result result = new Result(query); + result.hits().add(new Hit(name + ":1")); + return result; + } + + } + + private static class QueryCheckSearcher extends Searcher { + private static final String STATUS = "status"; + public static final String FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY = "FederationSearcher has cloned the query."; + public static final String OK = "Got the correct query."; + private final Query query; + + QueryCheckSearcher(final Query query) { + this.query = query; + } + + @Override + public Result search(final Query query, final Execution execution) { + final Result result = new Result(query); + if (query != this.query) { + result.hits().addError(ErrorMessage + .createErrorInPluginSearcher(FEDERATION_SEARCHER_HAS_CLONED_THE_QUERY)); + } else { + final Hit h = new Hit("QueryCheckSearcher status hit"); + h.setField(STATUS, OK); + result.hits().add(h); + } + return result; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java b/container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java new file mode 100644 index 00000000000..7b0451a01ba --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/FederationTester.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; + +import java.util.Collections; + +/** +* @author tonytv +*/ +class FederationTester { + SearchChainResolver.Builder builder = new SearchChainResolver.Builder(); + SearchChainRegistry registry = new SearchChainRegistry(); + + Execution execution; + + void addSearchChain(String id, Searcher... searchers) { + addSearchChain(id, federationOptions(), searchers); + } + + void addSearchChain(String id, FederationOptions federationOptions, Searcher... searchers) { + ComponentId searchChainId = ComponentId.fromString(id); + + builder.addSearchChain(searchChainId, federationOptions, Collections.<String>emptyList()); + + Chain<Searcher> chain = new Chain<>(searchChainId, searchers); + registry.register(chain); + } + + public void addOptionalSearchChain(String id, Searcher... searchers) { + addSearchChain(id, federationOptions().setOptional(true), searchers); + } + + private FederationOptions federationOptions() { + int preventTimeout = 24 * 60 * 60 * 1000; + return new FederationOptions().setUseByDefault(true).setTimeoutInMilliseconds(preventTimeout); + } + + FederationSearcher buildFederationSearcher() { + return new FederationSearcher(ComponentId.fromString("federation"), builder.build()); + } + + public Result search() { + return search(new Query()); + } + + public Result search(Query query) { + execution = createExecution(); + return execution.search(query); + } + + public Result searchAndFill() { + Result result = search(); + fill(result); + return result; + } + + private Execution createExecution() { + registry.freeze(); + return new Execution(new Chain<Searcher>(buildFederationSearcher()), Execution.Context.createContextStub(registry, null)); + } + + public void fill(Result result) { + execution.fill(result, "default"); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java new file mode 100644 index 00000000000..dcbbb217c7d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/HitCountTestCase.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.StringStartsWith.startsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class HitCountTestCase { + + @Test + public void require_that_offset_and_hits_are_adjusted_when_federating() { + final int chain1RelevanceMultiplier = 1; + final int chain2RelevanceMultiplier = 10; + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new AddHitsWithRelevanceSearcher("chain1", chain1RelevanceMultiplier)); + tester.addSearchChain("chain2", new AddHitsWithRelevanceSearcher("chain2", chain2RelevanceMultiplier)); + + Query query = new Query(); + query.setHits(5); + + query.setOffset(0); + assertAllHitsFrom("chain2", flattenAndTrim(tester.search(query))); + + query.setOffset(5); + assertAllHitsFrom("chain1", flattenAndTrim(tester.search(query))); + } + + @Test + public void require_that_hit_counts_are_merged() { + final long chain1TotalHitCount = 3; + final long chain1DeepHitCount = 5; + + final long chain2TotalHitCount = 7; + final long chain2DeepHitCount = 11; + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", new SetHitCountsSearcher(chain1TotalHitCount, chain1DeepHitCount)); + tester.addSearchChain("chain2", new SetHitCountsSearcher(chain2TotalHitCount, chain2DeepHitCount)); + + Result result = tester.searchAndFill(); + + assertThat(result.getTotalHitCount(), is(chain1TotalHitCount + chain2TotalHitCount)); + assertThat(result.getDeepHitCount(), is(chain1DeepHitCount + chain2DeepHitCount)); + } + + @Test + public void require_that_logging_hit_is_populated_with_result_count() { + final long chain1TotalHitCount = 9; + final long chain1DeepHitCount = 14; + + final long chain2TotalHitCount = 11; + final long chain2DeepHitCount = 15; + + FederationTester tester = new FederationTester(); + tester.addSearchChain("chain1", + new SetHitCountsSearcher(chain1TotalHitCount, chain1DeepHitCount)); + + tester.addSearchChain("chain2", + new SetHitCountsSearcher(chain2TotalHitCount, chain2DeepHitCount), + new AddHitsWithRelevanceSearcher("chain1", 2)); + + Query query = new Query(); + query.setOffset(2); + query.setHits(7); + Result result = tester.search(); + List<Hit> metaHits = getFirstMetaHitInEachGroup(result); + + Hit first = metaHits.get(0); + assertEquals(chain1TotalHitCount, first.getField("count_total")); + assertEquals(chain1TotalHitCount, first.getField("count_total")); + assertEquals(1, first.getField("count_first")); + assertEquals(0, first.getField("count_last")); + + Hit second = metaHits.get(1); + assertEquals(chain2TotalHitCount, second.getField("count_total")); + assertEquals(chain2TotalHitCount, second.getField("count_total")); + assertEquals(1, second.getField("count_first")); + assertEquals(AddHitsWithRelevanceSearcher.numHitsAdded, second.getField("count_last")); + + } + + private List<Hit> getFirstMetaHitInEachGroup(Result result) { + List<Hit> metaHits = new ArrayList<>(); + for (Hit topLevelHit : result.hits()) { + if (topLevelHit instanceof HitGroup) { + for (Hit hit : (HitGroup)topLevelHit) { + if (hit.isMeta()) { + metaHits.add(hit); + break; + } + } + } + } + return metaHits; + } + + private void assertAllHitsFrom(String chainName, HitGroup flattenedHits) { + for (Hit hit : flattenedHits) { + assertThat(hit.getId().toString(), startsWith(chainName)); + } + } + + private HitGroup flattenAndTrim(Result result) { + HitGroup flattenedHits = new HitGroup(); + result.setQuery(result.getQuery()); + flatten(result.hits(), flattenedHits); + + flattenedHits.trim(result.getQuery().getOffset(), result.getQuery().getHits()); + return flattenedHits; + } + + private void flatten(HitGroup hits, HitGroup flattenedHits) { + for (Hit hit : hits) { + if (hit instanceof HitGroup) { + flatten((HitGroup) hit, flattenedHits); + } else { + flattenedHits.add(hit); + } + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java b/container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java new file mode 100644 index 00000000000..81a3007735c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/test/SetHitCountsSearcher.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author tonytv + */ +class SetHitCountsSearcher extends Searcher { + + private final long totalHitCount; + private final long deepHitCount; + + public SetHitCountsSearcher(long totalHitCount, long deepHitCount) { + this.totalHitCount = totalHitCount; + this.deepHitCount = deepHitCount; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(createLoggingHit()); + + result.setTotalHitCount(totalHitCount); + result.setDeepHitCount(deepHitCount); + return result; + } + + private Hit createLoggingHit() { + Hit hit = new Hit("SetHitCountSearcher"); + hit.setMeta(true); + hit.types().add("logging"); + return hit; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java new file mode 100644 index 00000000000..2868d69457b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.*; +import com.yahoo.search.Query; +import com.yahoo.search.federation.vespa.QueryMarshaller; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class QueryMarshallerTestCase { + + private static final Linguistics linguistics = new SimpleLinguistics(); + + @Test + public void testCommonCommonCase() { + AndItem root = new AndItem(); + addThreeWords(root); + assertEquals("a AND b AND c", new QueryMarshaller().marshal(root)); + } + + @Test + public void testPhrase() { + PhraseItem root = new PhraseItem(); + root.setIndexName("habla"); + addThreeWords(root); + assertEquals("habla:\"a b c\"", new QueryMarshaller().marshal(root)); + } + + @Test + public void testPhraseDefaultIndex() { + PhraseItem root = new PhraseItem(); + addThreeWords(root); + assertEquals("\"a b c\"", new QueryMarshaller().marshal(root)); + } + + @Test + public void testLittleMoreComplex() { + AndItem root = new AndItem(); + addThreeWords(root); + OrItem ambig = new OrItem(); + root.addItem(ambig); + addThreeWords(ambig); + AndItem but = new AndItem(); + addThreeWords(but); + ambig.addItem(but); + assertEquals("a AND b AND c AND ( a OR b OR c OR ( a AND b AND c ) )", + new QueryMarshaller().marshal(root)); + } + + @Test + public void testRank() { + RankItem root = new RankItem(); + addThreeWords(root); + assertEquals("a RANK b RANK c", new QueryMarshaller().marshal(root)); + } + + @Test + public void testNear() { + NearItem near = new NearItem(3); + addThreeWords(near); + assertEquals("a NEAR(3) b NEAR(3) c", new QueryMarshaller().marshal(near)); + } + + @Test + public void testONear() { + ONearItem oNear = new ONearItem(3); + addThreeWords(oNear); + assertEquals("a ONEAR(3) b ONEAR(3) c", new QueryMarshaller().marshal(oNear)); + } + + private void addThreeWords(CompositeItem root) { + root.addItem(new WordItem("a")); + root.addItem(new WordItem("b")); + root.addItem(new WordItem("c")); + } + + @Test + public void testNegativeGroupedTerms() { + testQueryString(new QueryMarshaller(), "a -(b c) -(d e)", + "a ANDNOT ( b AND c ) ANDNOT ( d AND e )"); + } + + @Test + public void testPositiveGroupedTerms() { + testQueryString(new QueryMarshaller(), "a (b c)", "a AND ( b OR c )"); + } + + @Test + public void testInt() { + testQueryString(new QueryMarshaller(), "yahoo 123", "yahoo AND 123"); + } + + @Test + public void testCJKOneWord() { + testQueryString(new QueryMarshaller(), "天龍人"); + } + + @Test + public void testTwoWords() { + testQueryString(new QueryMarshaller(), "John Smith", "John AND Smith", null, new SimpleLinguistics()); + } + + @Test + public void testTwoWordsInPhrase() { + testQueryString(new QueryMarshaller(), "\"John Smith\"", "\"John Smith\"", null, new SimpleLinguistics()); + } + + @Test + public void testCJKTwoSentences() { + testQueryString(new QueryMarshaller(), "是不是這樣的夜晚 你才會這樣地想起我", "是不是這樣的夜晚 AND 你才會這樣地想起我"); + } + + @Test + public void testCJKTwoSentencesWithLanguage() { + testQueryString(new QueryMarshaller(), "助妳好孕 生1胎北市發2萬", "助妳好孕 AND 生1胎北市發2萬", "zh-Hant"); + } + + @Test + public void testCJKTwoSentencesInPhrase() { + QueryMarshaller marshaller = new QueryMarshaller(); + testQueryString(marshaller, "\"助妳好孕 生1胎北市發2萬\"", "\"助妳好孕 生1胎北市發2萬\"", "zh-Hant"); + testQueryString(marshaller, "\"是不是這樣的夜晚 你才會這樣地想起我\"", "\"是不是這樣的夜晚 你才會這樣地想起我\""); + } + + @Test + public void testCJKMultipleSentences() { + testQueryString(new QueryMarshaller(), "염부장님과 함께했던 좋은 추억들은", "염부장님과 AND 함께했던 AND 좋은 AND 추억들은"); + } + + @Test + public void testIndexRestriction() { + /** ticket 3707606, comment #29 */ + testQueryString(new QueryMarshaller(), "site:nytimes.com", "site:\"nytimes com\""); + } + + private void testQueryString(QueryMarshaller marshaller, String uq) { + testQueryString(marshaller, uq, uq, null); + } + + private void testQueryString(QueryMarshaller marshaller, String uq, String mq) { + testQueryString(marshaller, uq, mq, null); + } + + private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang) { + testQueryString(marshaller, uq, mq, lang, linguistics); + } + + private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang, Linguistics linguistics) { + Query query = new Query("/?query=" + QueryTestCase.httpEncode(uq) + ((lang != null) ? "&language=" + lang : "")); + query.getModel().setExecution(new Execution(new Execution.Context(null, new IndexFacts(), null, null, linguistics))); + assertEquals(mq, marshaller.marshal(query.getModel().getQueryTree().getRoot())); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java new file mode 100644 index 00000000000..9135984b26b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.search.Query; +import com.yahoo.search.federation.vespa.VespaSearcher; +import java.util.Map; + +/** + * Tests that source and backend specific parameters from the query are added correctly to the backend requests + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class QueryParametersTestCase extends junit.framework.TestCase { + + public void testQueryParameters() { + Query query=new Query(); + query.properties().set("a","a-value"); + query.properties().set("b.c","b.c-value"); + query.properties().set("source.otherSource.d","d-value"); + query.properties().set("source.testSource.e","e-value"); + query.properties().set("source.testSource.f.g","f.g-value"); + query.properties().set("provider.testProvider.h","h-value"); + query.properties().set("provider.testProvider.i.j","i.j-value"); + + query.properties().set("sourceName","testSource"); // Done by federation searcher + query.properties().set("providerName","testProvider"); // Done by federation searcher + + VespaSearcher searcher=new VespaSearcher("testProvider","",0,""); + Map<String,String> parameters=searcher.getQueryMap(query); + searcher.deconstruct(); + + assertEquals(9, parameters.size()); // 5 standard + the appropriate 4 of the above + assertEquals(parameters.get("e"),"e-value"); + assertEquals(parameters.get("f.g"),"f.g-value"); + assertEquals(parameters.get("h"),"h-value"); + assertEquals(parameters.get("i.j"),"i.j-value"); + } + +} + diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java new file mode 100644 index 00000000000..8cec6c64554 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import java.util.Iterator; + +import junit.framework.TestCase; + +import com.yahoo.net.URI; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.federation.vespa.ResultBuilder; +import com.yahoo.search.result.ErrorHit; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.HitGroup; + +/** + * Test XML parsing of results. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +@SuppressWarnings("deprecation") +public class ResultBuilderTestCase extends TestCase { + + public ResultBuilderTestCase (String name) { + super(name); + } + + private boolean quickCompare(double a, double b) { + double z = Math.min(Math.abs(a), Math.abs(b)); + if (Math.abs((a - b)) < (z / 1e14)) { + return true; + } else { + return false; + } + } + + public void testSimpleResult() { + boolean gotErrorDetails = false; + ResultBuilder r = new ResultBuilder(); + Result res = r.parse("file:src/test/java/com/yahoo/prelude/searcher/test/testhit.xml", new Query("?query=a")); + assertEquals(3, res.getConcreteHitCount()); + assertEquals(4, res.getHitCount()); + ErrorHit e = (ErrorHit) res.hits().get(0); + // known problem, if the same error is the main error is + // in details, it'll be added twice. Not sure how to fix that, + // because old Vespa systems give no error details, and there + // is no way of nuking an existing error if the details exist. + for (Iterator<?> i = e.errorIterator(); i.hasNext();) { + ErrorMessage err = (ErrorMessage) i.next(); + assertEquals(5, err.getCode()); + String details = err.getDetailedMessage(); + if (details != null) { + gotErrorDetails = true; + assertEquals("An error as ordered", details.trim()); + } + } + assertTrue("Error details are missing", gotErrorDetails); + assertEquals(new URI("http://def"), res.hits().get(1).getId()); + assertEquals("test/stuff\\tsome/other", res.hits().get(2).getField("category")); + assertEquals("<field>habla</field>" + + "<hi>blbl</hi><br /><>&fdlkkgj</field>;lk<a b=\"1\" c=\"2\" />" + + "<x><y><z /></y></x>", res.hits().get(3).getField("annoying").toString()); + } + + public void testNestedResult() { + ResultBuilder r = new ResultBuilder(); + Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml", new Query("?query=a")); + assertNull(res.hits().getError()); + assertEquals(3, res.hits().size()); + assertEquals("ABCDEFGHIJKLMNOPQRSTUVWXYZ", res.hits().get(0).getField("guid").toString()); + HitGroup g1 = (HitGroup) res.hits().get(1); + HitGroup g2 = (HitGroup) res.hits().get(2); + assertEquals(15, g1.size()); + assertEquals("reward_for_thumb", g1.get(1).getField("id").toString()); + assertEquals(10, g2.size()); + HitGroup g3 = (HitGroup) g2.get(3); + assertEquals("badge", g3.getTypeString()); + assertEquals(2, g3.size()); + assertEquals("badge/Topic Explorer 5", g3.get(0).getField("name").toString()); + } + + public void testWeirdDocumentID() { + ResultBuilder r = new ResultBuilder(); + Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml", new Query("?query=a")); + assertNull(res.hits().getError()); + assertEquals(3, res.hits().size()); + assertEquals(new URI("nalle"), res.hits().get(0).getId()); + assertEquals(new URI("tralle"), res.hits().get(1).getId()); + assertEquals(new URI("kalle"), res.hits().get(2).getId()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java new file mode 100644 index 00000000000..a1c3529e2e4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.vespa.VespaSearcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class VespaIntegrationTestCase extends junit.framework.TestCase { + + // TODO: Setup the answering vespa searcher from this test.... + public void testIt() { + if (System.currentTimeMillis() > 0) return; + Chain<Searcher> chain=new Chain<>(new VespaSearcher("test","example.yahoo.com",19010,"")); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=test")); + assertEquals(23,result.hits().size()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java new file mode 100644 index 00000000000..63da6adca77 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java @@ -0,0 +1,229 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.federation.vespa.test; + +import com.yahoo.prelude.query.*; +import com.yahoo.search.Query; +import com.yahoo.search.federation.vespa.VespaSearcher; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import junit.framework.TestCase; +import org.apache.http.HttpEntity; +import java.io.IOException; +import java.net.URI; + +/** + * Check query marshaling in VespaSearcher works... and stuff... + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class VespaSearcherTestCase extends TestCase { + + // TODO: More tests + + private VespaSearcher searcher; + + protected @Override void setUp() { + searcher = new VespaSearcher("cache1","",0,""); + } + + protected @Override void tearDown() { + searcher.deconstruct(); + } + + public void testMarshalQuery() { + RankItem root = new RankItem(); + QueryTree r = new QueryTree(root); + AndItem recall = new AndItem(); + PhraseItem usual = new PhraseItem(); + PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); + WordItem filterWord = new WordItem("silly"); + + filterPhrase.setFilter(true); + filterWord.setFilter(true); + + root.addItem(recall); + usual.addItem(new WordItem("new")); + usual.addItem(new WordItem("york")); + recall.addItem(usual); + recall.addItem(new WordItem("shoes")); + root.addItem(new WordItem("nike")); + root.addItem(new WordItem("adidas")); + root.addItem(filterPhrase); + recall.addItem(filterWord); + + assertEquals("( \"new york\" AND shoes AND silly ) RANK nike RANK adidas RANK \"bloody expensive\"", searcher.marshalQuery(r)); + } + + public void testMarshalQuerySmallTree() { + RankItem root = new RankItem(); + QueryTree r = new QueryTree(root); + AndItem recall = new AndItem(); + PhraseItem usual = new PhraseItem(); + PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); + WordItem filterWord = new WordItem("silly"); + + filterPhrase.setFilter(true); + filterWord.setFilter(true); + + root.addItem(recall); + usual.addItem(new WordItem("new")); + usual.addItem(new WordItem("york")); + recall.addItem(usual); + recall.addItem(new WordItem("shoes")); + root.addItem(filterPhrase); + recall.addItem(filterWord); + + assertEquals("( \"new york\" AND shoes AND silly ) RANK \"bloody expensive\"", searcher.marshalQuery(r)); + // TODO: Switch to this 2-way check rather than just 1-way and then also make this actually treat filter terms correctly + // assertMarshals(root) + } + + public void testWandMarshalling() { + WeakAndItem root = new WeakAndItem(); + root.setN(32); + root.addItem(new WordItem("a")); + root.addItem(new WordItem("b")); + root.addItem(new WordItem("c")); + assertMarshals(root); + } + + public void testWandMarshalling2() { + // AND (WAND(10) a!1 the!10) source:yahoonews + AndItem root = new AndItem(); + WeakAndItem wand = new WeakAndItem(10); + wand.addItem(newWeightedWordItem("a",1)); + wand.addItem(newWeightedWordItem("the",10)); + root.addItem(wand); + root.addItem(new WordItem("yahoonews","source")); + assertMarshals(root); + } + + private WordItem newWeightedWordItem(String word,int weight) { + WordItem wordItem=new WordItem(word); + wordItem.setWeight(weight); + return wordItem; + } + + private void assertMarshals(Item root) { + QueryTree r = new QueryTree(root); + String marshalledQuery=searcher.marshalQuery(r); + assertEquals("Marshalled form '" + marshalledQuery + "' recreates the original", + r,parseQuery(marshalledQuery,"")); + } + + private static Item parseQuery(String query, String filter) { + Parser parser = ParserFactory.newInstance(Query.Type.ADVANCED, new ParserEnvironment()); + return parser.parse(new Parsable().setQuery(query).setFilter(filter)); + } + + public void testSourceProviderProperties() throws Exception { + /* TODO: update test + Server httpServer = new Server(); + try { + SocketConnector listener = new SocketConnector(); + listener.setHost("0.0.0.0"); + httpServer.addConnector(listener); + httpServer.setHandler(new DummyHandler()); + httpServer.start(); + + int port=httpServer.getConnectors()[0].getLocalPort(); + + List<SourcesConfig.Source> sourcesConfig = new ArrayList<SourcesConfig.Source>(); + SourcesConfig.Source sourceConfig = new SourcesConfig.Source(); + sourceConfig.chain.setValue("news"); + sourceConfig.provider.setValue("news"); + sourceConfig.id.setValue("news"); + sourceConfig.timelimit.value = 10000; + sourcesConfig.add(sourceConfig); + FederationSearcher federator = + new FederationSearcher(ComponentId.createAnonymousComponentId(), + new ArrayList<SourcesConfig.Source>(sourcesConfig)); + SearchChain mainChain=new OrderedSearchChain(federator); + + SearchChainRegistry registry=new SearchChainRegistry(); + SearchChain sourceChain=new SearchChain(new ComponentId("news"),new VespaSearcher("test","localhost",port,"")); + registry.register(sourceChain); + Query query=new Query("?query=hans&hits=20&provider.news.a=a1&source.news.b=b1"); + Result result=new Execution(mainChain,registry).search(query); + assertNull(result.hits().getError()); + Hit testHit=result.hits().get("testHit"); + assertNotNull(testHit); + assertEquals("testValue",testHit.fields().get("testField")); + assertEquals("a1",testHit.fields().get("a")); + assertEquals("b1",testHit.fields().get("b")); + } + finally { + httpServer.stop(); + } + */ + } + + public void testVespaSearcher() { + VespaSearcher v=new VespaSearcherValidatingSubclass(); + new Execution(v, Execution.Context.createContextStub()).search(new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&filter=myfilter"))); + } + + private class VespaSearcherValidatingSubclass extends VespaSearcher { + + public VespaSearcherValidatingSubclass() { + super("configId","host",80,"path"); + } + + @Override + protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { + assertEquals("http://host:80/path?query=test+RANK+myfilter&type=adv&offset=0&hits=10&presentation.format=xml",uri.toString()); + return super.getEntity(uri,requestMeta,query); + } + + } + + // used by the old testSourceProviderProperties() +// private class DummyHandler extends AbstractHandler { +// public void handle(String s, Request request, HttpServletRequest httpServletRequest, +// HttpServletResponse httpServletResponse) throws IOException, ServletException { +// +// try { +// Response httpResponse = httpServletResponse instanceof Response ? (Response) httpServletResponse : HttpConnection.getCurrentConnection().getResponse(); +// +// httpResponse.setStatus(HttpStatus.OK_200); +// httpResponse.setContentType("text/xml"); +// httpResponse.setCharacterEncoding("UTF-8"); +// Result r=new Result(new Query()); +// Hit testHit=new Hit("testHit"); +// testHit.setField("uri","testHit"); // That this is necessary is quite unfortunate... +// testHit.setField("testField","testValue"); +// // Write back all incoming properties: +// for (Object e : httpServletRequest.getParameterMap().entrySet()) { +// Map.Entry entry=(Map.Entry)e; +// testHit.setField(entry.getKey().toString(),getFirstValue(entry.getValue())); +// } +// +// r.hits().add(testHit); +// +// //StringWriter sw=new StringWriter(); +// //r.render(sw); +// //System.out.println(sw.toString()); +// +// SearchRendererAdaptor.callRender(httpResponse.getWriter(), r); +// httpResponse.complete(); +// } +// catch (Exception e) { +// System.out.println("WARNING: Could not respond to request: " + Exceptions.toMessageString(e)); +// e.printStackTrace(); +// } +// } +// +// private String getFirstValue(Object entry) { +// if (entry instanceof String[]) +// return ((String[])entry)[0].toString(); +// else +// return entry.toString(); +// } +// } + +} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml new file mode 100644 index 00000000000..b4b5c072eca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result total-hit-count="3"> + <hit relevancy="75" source="test" type="summary"> + <field name="uri">nalle</field> + <field name="relevancy">75</field> + <field name="collapseId">0</field> + </hit> + <hit relevancy="73" source="test" type="summary test other"> + <field name="documentId">tralle</field> + <field name="relevancy">73</field> + <field name="collapseId">0</field> + <field name="category">test/stuff\tsome/other</field> + <field name="bsumtitle">dklf øæå sdf > & < +Ipsum, etc.</field> + </hit> + <hit relevancy="70" source="test" type="summary"> + <field name="DOCUMENTID">kalle</field> + <field name="relevancy">75</field> + <field name="collapseId">0</field> + <field name="annoying"><field>habla</field><hi>blbl</hi><br /><![CDATA[<>&fdlkkgj</field>]]>;lk<a b="1" c="2" /><x><y><z /></y></x></field> + </hit> +</result> diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml new file mode 100644 index 00000000000..c935f16528f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml @@ -0,0 +1,318 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result total-hit-count="36"> +<hit type="user_reputation"> +<field name="guid">ABCDEFGHIJKLMNOPQRSTUVWXYZ</field> +<field name="level">zero</field> +<field name="points">0</field> +<field name="created">1287600988</field> +<field name="updated">1287600988</field> +</hit> +<group type="actions"> +<hit type="action"> +<field name="id">thumb</field> +<field name="created">1287600992</field> +<field name="updated">1287600992</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> +<hit type="action"> +<field name="id">reward_for_thumb</field> +<field name="created">1287600992</field> +<field name="updated">1287600992</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> +<hit type="action"> +<field name="id">undo_thumb</field> +<field name="created">1287600992</field> +<field name="updated">1287600992</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">buzz</field> +<field name="created">1287600989</field> +<field name="updated">1287600989</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">undo_reward_for_thumb</field> +<field name="created">1287600992</field> +<field name="updated">1287600992</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">vote</field> +<field name="created">1287600989</field> +<field name="updated">1287600989</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">report_abuse</field> +<field name="created">1287600992</field> +<field name="updated">1287600992</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">reward_for_vote</field> +<field name="created">1287600989</field> +<field name="updated">1287600989</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">signup</field> +<field name="created">1287600993</field> +<field name="updated">1287600993</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">registered</field> +<field name="created">1287600989</field> +<field name="updated">1287600989</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">get_points</field> +<field name="created">1287600989</field> +<field name="updated">1287600989</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">contrib_SignedUp</field> +<field name="created">1287600993</field> +<field name="updated">1287600993</field> +<field name="points">0</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">contrib_AgreedToTos</field> +<field name="created">1287600993</field> +<field name="updated">1287600993</field> +<field name="points">500</field> +<field name="level">zero</field> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">Create Feature</field> +<field name="created"/> +<field name="updated"/> +<field name="points">0</field> +<field name="level"/> +<field name="isEnabled">1</field> +</hit> + +<hit type="action"> +<field name="id">add_theme</field> +<field name="created"/> +<field name="updated"/> +<field name="points">0</field> +<field name="level"/> +<field name="isEnabled">1</field> +</hit> +</group> + +<group type="awards"> + +<group type="badge"> + +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/First Feature</field> +<field name="description">You’ve created your First Feature!</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/1stfeature.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> + +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> + +<group type="badge"> + +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/25th Feature</field> +<field name="description">You’ve created your 25th Feature!</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/25thfeature.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> + +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> + +<group type="badge"> + +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/50th Feature</field> +<field name="description">You’ve created your 50th Feature!</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/10thfeature.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> + +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> + +<group type="badge"> + +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/Topic Explorer 5</field> +<field name="description">You’ve added a Feature to your 5th Topic Page!</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/5thtopic.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> + +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> + +<group type="badge"> + +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/Topic Explorer 15</field> +<field name="description">You’ve added a Feature to your 15th Topic Page!</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/15thtopic.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> + +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> + +<group type="badge"> + +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/Topic Explorer 30</field> +<field name="description">You’ve added a Feature to your 30th Topic Page!</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/30thtopic.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> + +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> + +<group type="badge"> + +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/Pollster</field> +<field name="description">You’ve created your 5th Poll Feature.</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/pollster.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> +<group type="badge"> +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/Reporter</field> +<field name="description">You’ve created your 5th Article Feature.</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/newsreporter.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> +<group type="badge"> +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/Paparazzi</field> +<field name="description">You’ve created your 5th Image Feature.</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/paparazzi.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> +<group type="badge"> +<hit type="info"> +<field name="type">badge</field> +<field name="name">badge/Video Reporter</field> +<field name="description">You’ve created your 5th Video Feature.</field> +<field name="status">active</field> +<field name="imageUrl">http://example.yahoo.com/director.png</field> +<field name="imageHeight">57</field> +<field name="imageWidth">57</field> +</hit> +<hit type="earned"> +<field name="date">1283981088</field> +<field name="context">topic/Jennifer_Aniston</field> +</hit> +</group> +</group> +</result> diff --git a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore diff --git a/container-search/src/test/java/com/yahoo/search/grouping/ContinuationTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/ContinuationTestCase.java new file mode 100644 index 00000000000..bc0d69f4bf7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/ContinuationTestCase.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ContinuationTestCase { + + private static final String KNOWN_CONTINUATION = "BCBCBCBEBGBCBKCBACBKCCK"; + + @Test + public void requireThatToStringCanBeParsedByFromString() { + Continuation cnt = Continuation.fromString(KNOWN_CONTINUATION); + assertNotNull(cnt); + assertEquals(KNOWN_CONTINUATION, cnt.toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java new file mode 100644 index 00000000000..19723dcd51a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingQueryParserTestCase.java @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import com.yahoo.search.Query; +import com.yahoo.search.grouping.request.AllOperation; +import com.yahoo.search.grouping.request.EachOperation; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.searchchain.Execution; + +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.TimeZone; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupingQueryParserTestCase { + + @Test + public void requireThatNoRequestIsSkipped() { + assertEquals(Collections.emptyList(), executeQuery(null, null, null)); + } + + @Test + public void requireThatEmptyRequestIsSkipped() { + assertEquals(Collections.emptyList(), executeQuery("", null, null)); + } + + @Test + public void requireThatRequestIsParsed() { + List<GroupingRequest> lst = executeQuery("all(group(foo) each(output(max(bar))))", null, null); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + assertNotNull(req.getRootOperation()); + } + + @Test + public void requireThatRequestListIsParsed() { + List<GroupingRequest> lst = executeQuery("all();each()", null, null); + assertNotNull(lst); + assertEquals(2, lst.size()); + assertTrue(lst.get(0).getRootOperation() instanceof AllOperation); + assertTrue(lst.get(1).getRootOperation() instanceof EachOperation); + } + + @Test + public void requireThatEachRightBelowAllParses() { + List<GroupingRequest> lst = executeQuery("all(each(output(summary(bar))))", + null, null); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + final GroupingOperation rootOperation = req.getRootOperation(); + assertNotNull(rootOperation); + assertSame(AllOperation.class, rootOperation.getClass()); + assertSame(EachOperation.class, rootOperation.getChildren().get(0).getClass()); + } + + @Test + public void requireThatContinuationListIsParsed() { + List<GroupingRequest> lst = executeQuery("all(group(foo) each(output(max(bar))))", + "BCBCBCBEBGBCBKCBACBKCCK BCBBBBBDBF", null); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + assertNotNull(req.getRootOperation()); + assertEquals(2, req.continuations().size()); + } + + @Test + public void requireThatTimeZoneIsParsed() { + List<GroupingRequest> lst = executeQuery("all(group(foo) each(output(max(bar))))", null, "cet"); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + TimeZone time = req.getTimeZone(); + assertNotNull(time); + assertEquals(TimeZone.getTimeZone("cet"), time); + } + + @Test + public void requireThatTimeZoneHasUtcDefault() { + List<GroupingRequest> lst = executeQuery("all(group(foo) each(output(max(bar))))", null, null); + assertNotNull(lst); + assertEquals(1, lst.size()); + GroupingRequest req = lst.get(0); + assertNotNull(req); + TimeZone time = req.getTimeZone(); + assertNotNull(time); + assertEquals(TimeZone.getTimeZone("utc"), time); + } + + private static List<GroupingRequest> executeQuery(String request, String continuation, String timeZone) { + Query query = new Query(); + query.properties().set(GroupingQueryParser.PARAM_REQUEST, request); + query.properties().set(GroupingQueryParser.PARAM_CONTINUE, continuation); + query.properties().set(GroupingQueryParser.PARAM_TIMEZONE, timeZone); + new Execution(new GroupingQueryParser(), Execution.Context.createContextStub()).search(query); + return GroupingRequest.getRequests(query); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java new file mode 100644 index 00000000000..38e94092644 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingRequestTestCase.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.grouping.result.Group; +import com.yahoo.search.grouping.result.RootGroup; +import com.yahoo.search.result.Hit; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupingRequestTestCase { + + @Test + public void requireThatContinuationListIsMutable() { + GroupingRequest req = GroupingRequest.newInstance(new Query()); + assertTrue(req.continuations().isEmpty()); + + Continuation foo = new Continuation() { + + }; + req.continuations().add(foo); + assertEquals(Arrays.asList(foo), req.continuations()); + + req.continuations().clear(); + assertTrue(req.continuations().isEmpty()); + } + + @Test + public void requireThatResultIsFound() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + Result res = new Result(query); + + res.hits().add(new Hit("foo")); + RootGroup bar = newRootGroup(0); + req.setResultGroup(bar); + res.hits().add(bar); + res.hits().add(new Hit("baz")); + + Group grp = req.getResultGroup(res); + assertNotNull(grp); + assertSame(bar, grp); + } + + @Test + public void requireThatParallelRequestsAreSupported() { + Query query = new Query(); + Result res = new Result(query); + + GroupingRequest reqA = GroupingRequest.newInstance(query); + RootGroup grpA = newRootGroup(0); + reqA.setResultGroup(grpA); + res.hits().add(grpA); + + GroupingRequest reqB = GroupingRequest.newInstance(query); + RootGroup grpB = newRootGroup(1); + reqB.setResultGroup(grpB); + res.hits().add(grpB); + + Group grp = reqA.getResultGroup(res); + assertNotNull(grp); + assertSame(grpA, grp); + + assertNotNull(grp = reqB.getResultGroup(res)); + assertSame(grpB, grp); + } + + @Test + public void requireThatRemovedResultIsNull() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + Result res = new Result(query); + + res.hits().add(new Hit("foo")); + RootGroup bar = newRootGroup(0); + req.setResultGroup(bar); + res.hits().add(new Hit("baz")); + + assertNull(req.getResultGroup(res)); + } + + @Test + public void requireThatNonGroupResultIsNull() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + Result res = new Result(query); + + RootGroup grp = newRootGroup(0); + req.setResultGroup(grp); + res.hits().add(new Hit(grp.getId().toString())); + + assertNull(req.getResultGroup(res)); + } + + @Test + public void requireThatGetRequestsReturnsAllRequests() { + Query query = new Query(); + assertEquals(Collections.emptyList(), GroupingRequest.getRequests(query)); + + GroupingRequest foo = GroupingRequest.newInstance(query); + assertEquals(Arrays.asList(foo), GroupingRequest.getRequests(query)); + + GroupingRequest bar = GroupingRequest.newInstance(query); + assertEquals(Arrays.asList(foo, bar), GroupingRequest.getRequests(query)); + } + + @Test + public void requireThatGetRequestThrowsIllegalArgumentOnBadProperty() throws Exception { + Query query = new Query(); + Field propName = GroupingRequest.class.getDeclaredField("PROP_REQUEST"); + propName.setAccessible(true); + query.properties().set((CompoundName)propName.get(null), new Object()); + try { + GroupingRequest.getRequests(query); + fail(); + } catch (IllegalArgumentException e) { + + } + } + + private static RootGroup newRootGroup(int id) { + return new RootGroup(id, new Continuation() { + + }); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java new file mode 100644 index 00000000000..38248bad6cf --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/GroupingValidatorTestCase.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.Query; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupingValidatorTestCase { + + @Test + public void requireThatAvailableAttributesDoNotThrow() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + validateGrouping("myCluster", Arrays.asList("foo", "bar"), query); + } + + @Test + public void requireThatUnavailableAttributesThrow() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + try { + validateGrouping("myCluster", Arrays.asList("foo"), query); + fail("Validator should throw exception because attribute 'bar' is unavailable."); + } catch (UnavailableAttributeException e) { + assertEquals("myCluster", e.getClusterName()); + assertEquals("bar", e.getAttributeName()); + } + } + + @Test + public void requireThatEnableFlagPreventsThrow() { + Query query = new Query(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + query.properties().set(GroupingValidator.PARAM_ENABLED, "false"); + validateGrouping("myCluster", Arrays.asList("foo"), query); + } + + private static void validateGrouping(String clusterName, Collection<String> attributeNames, Query query) { + QrSearchersConfig.Builder qrsConfig = new QrSearchersConfig.Builder().searchcluster( + new QrSearchersConfig.Searchcluster.Builder() + .indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.REALTIME) + .name(clusterName)); + ClusterConfig.Builder clusterConfig = new ClusterConfig.Builder(). + clusterId(0). + clusterName("test"); + AttributesConfig.Builder attributesConfig = new AttributesConfig.Builder(); + for (String attributeName : attributeNames) { + attributesConfig.attribute(new AttributesConfig.Attribute.Builder() + .name(attributeName)); + } + new Execution( + new GroupingValidator(new QrSearchersConfig(qrsConfig), + new ClusterConfig(clusterConfig), + new AttributesConfig(attributesConfig)), + Execution.Context.createContextStub()).search(query); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java new file mode 100644 index 00000000000..c674a8a0755 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/UniqueGroupingSearcherTestCase.java @@ -0,0 +1,219 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.result.Group; +import com.yahoo.search.grouping.result.GroupList; +import com.yahoo.search.grouping.result.HitList; +import com.yahoo.search.grouping.result.RootGroup; +import com.yahoo.search.grouping.result.StringId; +import com.yahoo.search.query.Sorting; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.yolean.Exceptions; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * @author <a href="http://techyard.corp.yahoo-inc.com/en/user/andreer">Andreas Eriksen</a> + */ +public class UniqueGroupingSearcherTestCase { + + @Test + public void testSkipGroupingBasedDedup() throws Exception { + Result result = search("?query=foo", + new MockResultProvider(0, false)); + assertEquals(0, result.hits().size()); + } + + @Test + public void testSkipGroupingBasedDedupIfMultiLevelSorting() throws Exception { + Result result = search("?query=foo&unique=fingerprint&sorting=-pubdate%20-[rank]", + new MockResultProvider(0, false)); + assertEquals(0, result.hits().size()); + } + @Test + public void testIllegalSortingSpec() { + try { + search("?query=foo&unique=fingerprint&sorting=-1", + new MockResultProvider(0, true).addGroupList(new GroupList("fingerprint"))); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString( + "Invalid request parameter: Could not set 'ranking.sorting' to '-1': " + + "Illegal attribute name '1' for sorting. Requires '[\\[]*[a-zA-Z_][\\.a-zA-Z0-9_-]*[\\]]*'")); + } + } + + @Test + public void testGroupingBasedDedupNoGroupingHits() throws Exception { + Result result = search("?query=foo&unique=fingerprint", + new MockResultProvider(0, true)); + assertEquals(0, result.hits().size()); + } + + @Test + public void testGroupingBasedDedupWithEmptyGroupingHitsList() throws Exception { + Result result = search("?query=foo&unique=fingerprint", + new MockResultProvider(0, true).addGroupList(new GroupList("fingerprint"))); + assertEquals(0, result.hits().size()); + assertEquals(0, result.getTotalHitCount()); + } + + @Test + public void testGroupingBasedDedupWithNullGroupingResult() throws Exception { + try { + search("?query=foo&unique=fingerprint", + new MockResultProvider(0, false)); + fail(); + } catch (IllegalStateException e) { + assertEquals("Failed to produce deduped result set.", e.getMessage()); + } + } + + @Test + public void testGroupingBasedDedupWithGroupingHits() throws Exception { + GroupList fingerprint = new GroupList("fingerprint"); + fingerprint.add(makeHitGroup("1")); + fingerprint.add(makeHitGroup("2")); + fingerprint.add(makeHitGroup("3")); + fingerprint.add(makeHitGroup("4")); + fingerprint.add(makeHitGroup("5")); + fingerprint.add(makeHitGroup("6")); + fingerprint.add(makeHitGroup("7")); + + MockResultProvider mockResultProvider = new MockResultProvider(15, true); + mockResultProvider.addGroupList(fingerprint); + mockResultProvider.resultGroup.setField(UniqueGroupingSearcher.LABEL_COUNT, 42l); + Result result = search("?query=foo&unique=fingerprint&hits=5&offset=1", mockResultProvider); + assertEquals(5, result.hits().size()); + assertEquals("2", result.hits().get(0).getId().toString()); + assertEquals("3", result.hits().get(1).getId().toString()); + assertEquals("4", result.hits().get(2).getId().toString()); + assertEquals("5", result.hits().get(3).getId().toString()); + assertEquals("6", result.hits().get(4).getId().toString()); + assertEquals(42, result.getTotalHitCount()); + } + + @Test + public void testGroupingBasedDedupWithGroupingHitsAndSorting() throws Exception { + GroupList fingerprint = new GroupList("fingerprint"); + fingerprint.add(makeSortingHitGroup("1")); + fingerprint.add(makeSortingHitGroup("2")); + fingerprint.add(makeSortingHitGroup("3")); + fingerprint.add(makeSortingHitGroup("4")); + fingerprint.add(makeSortingHitGroup("5")); + fingerprint.add(makeSortingHitGroup("6")); + fingerprint.add(makeSortingHitGroup("7")); + + MockResultProvider mockResultProvider = new MockResultProvider(100, true); + mockResultProvider.addGroupList(fingerprint); + mockResultProvider.resultGroup.setField(UniqueGroupingSearcher.LABEL_COUNT, 1337l); + + Result result = search("?query=foo&unique=fingerprint&hits=5&offset=1&sorting=-expdate", mockResultProvider); + assertEquals(5, result.hits().size()); + assertEquals("2", result.hits().get(0).getId().toString()); + assertEquals("3", result.hits().get(1).getId().toString()); + assertEquals("4", result.hits().get(2).getId().toString()); + assertEquals("5", result.hits().get(3).getId().toString()); + assertEquals("6", result.hits().get(4).getId().toString()); + assertEquals(1337, result.getTotalHitCount()); + } + + @Test + public void testBuildGroupingExpression() throws Exception { + assertEquals("all(group(title) max(11) output(count() as(uniqueCount)) each(max(1) each(output(summary())) " + + "as(uniqueHits)))", + UniqueGroupingSearcher + .buildGroupingExpression("title", 11, null, null) + .toString()); + assertEquals("all(group(fingerprint) max(5) output(count() as(uniqueCount)) each(max(1) " + + "each(output(summary(attributeprefetch))) as(uniqueHits)))", + UniqueGroupingSearcher + .buildGroupingExpression("fingerprint", 5, "attributeprefetch", null) + .toString()); + assertEquals("all(group(fingerprint) max(5) order(neg(max(pubdate))) output(count() as(uniqueCount)) each(" + + "all(group(neg(pubdate)) max(1) order(neg(max(pubdate))) each(each(output(summary())) " + + "as(uniqueHits)) as(uniqueGroups))))", + UniqueGroupingSearcher + .buildGroupingExpression("fingerprint", 5, null, new Sorting("-pubdate")) + .toString()); + assertEquals("all(group(fingerprint) max(5) order(min(pubdate)) output(count() as(uniqueCount)) each(" + + "all(group(pubdate) max(1) order(min(pubdate)) each(each(output(summary(attributeprefetch))) " + + "as(uniqueHits)) as(uniqueGroups))))", + UniqueGroupingSearcher + .buildGroupingExpression("fingerprint", 5, "attributeprefetch", new Sorting("+pubdate")) + .toString()); + } + + private static Group makeHitGroup(String name) { + Group ein = new Group(new StringId(name), new Relevance(0)); + HitList hits = new HitList(UniqueGroupingSearcher.LABEL_HITS); + hits.add(new Hit(name)); + ein.add(hits); + return ein; + } + + private static Group makeSortingHitGroup(String name) { + Hit hit = new Hit(name); + + HitList hits = new HitList(UniqueGroupingSearcher.LABEL_HITS); + hits.add(hit); + + Group dedupGroup = new Group(new StringId(name), new Relevance(0)); + dedupGroup.add(hits); + + GroupList dedupedHits = new GroupList(UniqueGroupingSearcher.LABEL_GROUPS); + dedupedHits.add(dedupGroup); + + Group ein = new Group(new StringId(name), new Relevance(0)); + ein.add(dedupedHits); + return ein; + } + + private static Result search(String query, MockResultProvider result) { + return new Execution(new Chain<>(new UniqueGroupingSearcher(), result), + Execution.Context.createContextStub()).search(new Query(query)); + } + + private static class MockResultProvider extends Searcher { + + final RootGroup resultGroup; + final long totalHitCount; + final boolean addGroupingData; + + MockResultProvider(long totalHitCount, boolean addGroupingData) { + this.addGroupingData = addGroupingData; + this.resultGroup = new RootGroup(0, null); + this.totalHitCount = totalHitCount; + } + + MockResultProvider addGroupList(GroupList groupList) { + resultGroup.add(groupList); + return this; + } + + @Override + public Result search(Query query, Execution execution) { + Result result = new Result(query); + if (addGroupingData) { + result.hits().add(resultGroup); + GroupingRequest.getRequests(query).get(0).setResultGroup(resultGroup); + result.setTotalHitCount(totalHitCount); + } + return result; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java new file mode 100644 index 00000000000..0ee23a3f37f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/BucketResolverTestCase.java @@ -0,0 +1,212 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import java.text.ChoiceFormat; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@SuppressWarnings({ "rawtypes" }) +public class BucketResolverTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests + // + // -------------------------------------------------------------------------------- + + @Test + public void testResolve() { + BucketResolver resolver = new BucketResolver(); + resolver.push(new StringValue("a"), true); + try { + resolver.resolve(new AttributeValue("foo")); + fail(); + } catch (IllegalStateException e) { + assertEquals("Missing to-limit of last bucket.", e.getMessage()); + } + + resolver.push(new StringValue("b"), false); + PredefinedFunction fnc = resolver.resolve(new AttributeValue("foo")); + assertNotNull(fnc); + assertEquals(1, fnc.getNumBuckets()); + BucketValue exp = fnc.getBucket(0); + assertNotNull(exp); + assertTrue(exp.getFrom() instanceof StringValue); + assertTrue(exp.getTo() instanceof StringValue); + BucketValue val = exp; + assertEquals("a", val.getFrom().getValue()); + assertEquals("b", val.getTo().getValue()); + + resolver.push(new StringValue("c"), true); + try { + resolver.resolve(new AttributeValue("foo")); + fail(); + } catch (IllegalStateException e) { + assertEquals("Missing to-limit of last bucket.", e.getMessage()); + } + + resolver.push(new StringValue("d"), false); + fnc = resolver.resolve(new AttributeValue("foo")); + assertNotNull(fnc); + assertEquals(2, fnc.getNumBuckets()); + assertNotNull(exp = fnc.getBucket(0)); + assertTrue(exp.getFrom() instanceof StringValue); + assertTrue(exp.getTo() instanceof StringValue); + val = exp; + assertEquals("a", val.getFrom().getValue()); + assertEquals("b", val.getTo().getValue()); + assertNotNull(exp = fnc.getBucket(1)); + assertTrue(exp.getFrom() instanceof StringValue); + assertTrue(exp.getTo() instanceof StringValue); + val = exp; + assertEquals("c", val.getFrom().getValue()); + assertEquals("d", val.getTo().getValue()); + } + + @Test + public void testBucketType() { + checkPushFail(Arrays.asList((ConstantValue)new StringValue("a"), new LongValue(1L)), + "Bucket type mismatch, expected 'StringValue' got 'LongValue'."); + checkPushFail(Arrays.asList((ConstantValue)new StringValue("a"), new DoubleValue(1.0)), + "Bucket type mismatch, expected 'StringValue' got 'DoubleValue'."); + checkPushFail(Arrays.asList((ConstantValue)new LongValue(1L), new StringValue("a")), + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + checkPushFail(Arrays.asList((ConstantValue)new LongValue(1L), new DoubleValue(1.0)), + "Bucket type mismatch, expected 'LongValue' got 'DoubleValue'."); + checkPushFail(Arrays.asList((ConstantValue)new DoubleValue(1.0), new StringValue("a")), + "Bucket type mismatch, expected 'DoubleValue' got 'StringValue'."); + checkPushFail(Arrays.asList((ConstantValue)new DoubleValue(1.0), new LongValue(1L)), + "Bucket type mismatch, expected 'DoubleValue' got 'LongValue'."); + checkPushFail(Arrays.asList((ConstantValue)new InfiniteValue(new Infinite(true)), new InfiniteValue(new Infinite(false))), + "Bucket type mismatch, cannot both be infinity."); + + } + + @Test + public void testBucketOrder() { + checkPushFail(Arrays.asList((ConstantValue)new LongValue(2L), new LongValue(1L)), + "Bucket to-value can not be less than from-value."); + checkPushFail(Arrays.asList((ConstantValue)new DoubleValue(2.0), new DoubleValue(1.0)), + "Bucket to-value can not be less than from-value."); + checkPushFail(Arrays.asList((ConstantValue)new StringValue("b"), new StringValue("a")), + "Bucket to-value can not be less than from-value."); + } + + public void assertBucketRange(BucketValue expected, ConstantValue from, boolean inclusiveFrom, ConstantValue to, boolean inclusiveTo) { + BucketResolver resolver = new BucketResolver(); + resolver.push(from, inclusiveFrom); + resolver.push(to, inclusiveTo); + PredefinedFunction fnc = resolver.resolve(new AttributeValue("foo")); + assertNotNull(fnc); + BucketValue result = fnc.getBucket(0); + assertEquals(result.getFrom().getValue(), expected.getFrom().getValue()); + assertEquals(result.getTo().getValue(), expected.getTo().getValue()); + } + + public void assertBucketOrder(BucketResolver resolver) { + PredefinedFunction fnc = resolver.resolve(new AttributeValue("foo")); + BucketValue prev = null; + for (int i = 0; i < fnc.getNumBuckets(); i++) { + BucketValue b = fnc.getBucket(i); + if (prev != null) { + assertTrue(prev.compareTo(b) < 0); + } + prev = b; + } + } + + @Test + public void requireThatBucketRangesWork() { + BucketValue expected = new LongBucket(2, 5); + assertBucketRange(expected, new LongValue(1), false, new LongValue(4), true); + assertBucketRange(expected, new LongValue(1), false, new LongValue(5), false); + assertBucketRange(expected, new LongValue(2), true, new LongValue(4), true); + assertBucketRange(expected, new LongValue(2), true, new LongValue(5), false); + + + BucketResolver resolver = new BucketResolver(); + resolver.push(new LongValue(1), true).push(new LongValue(2), false); + resolver.push(new LongValue(2), true).push(new LongValue(4), true); + resolver.push(new LongValue(4), false).push(new LongValue(5), false); + resolver.push(new LongValue(5), false).push(new LongValue(8), true); + assertBucketOrder(resolver); + + + expected = new StringBucket("aba ", "bab "); + assertBucketRange(expected, new StringValue("aba"), false, new StringValue("bab"), true); + assertBucketRange(expected, new StringValue("aba"), false, new StringValue("bab "), false); + assertBucketRange(expected, new StringValue("aba "), true, new StringValue("bab"), true); + assertBucketRange(expected, new StringValue("aba "), true, new StringValue("bab "), false); + + resolver = new BucketResolver(); + resolver.push(new StringValue("aaa"), true).push(new StringValue("aab"), false); + resolver.push(new StringValue("aab"), true).push(new StringValue("aac"), true); + resolver.push(new StringValue("aac"), false).push(new StringValue("aad"), false); + resolver.push(new StringValue("aad"), false).push(new StringValue("aae"), true); + assertBucketOrder(resolver); + + RawBuffer r1 = new RawBuffer(new byte[]{0, 1, 3}); + RawBuffer r1next = new RawBuffer(new byte[]{0, 1, 3, 0}); + RawBuffer r2 = new RawBuffer(new byte[]{0, 2, 2}); + RawBuffer r2next = new RawBuffer(new byte[]{0, 2, 2, 0}); + RawBuffer r2nextnext = new RawBuffer(new byte[]{0, 2, 2, 0, 4}); + + expected = new RawBucket(r1next, r2next); + assertBucketRange(expected, new RawValue(r1), false, new RawValue(r2), true); + assertBucketRange(expected, new RawValue(r1), false, new RawValue(r2next), false); + assertBucketRange(expected, new RawValue(r1next), true, new RawValue(r2), true); + assertBucketRange(expected, new RawValue(r1next), true, new RawValue(r2next), false); + + resolver = new BucketResolver(); + resolver.push(new RawValue(r1), true).push(new RawValue(r1next), false); + resolver.push(new RawValue(r1next), true).push(new RawValue(r2), true); + resolver.push(new RawValue(r2), false).push(new RawValue(r2next), false); + resolver.push(new RawValue(r2next), false).push(new RawValue(r2nextnext), true); + assertBucketOrder(resolver); + + double d1next = ChoiceFormat.nextDouble(1.414); + double d2next = ChoiceFormat.nextDouble(3.14159); + double d1 = ChoiceFormat.nextDouble(d1next); + double d2 = ChoiceFormat.nextDouble(d2next); + expected = new DoubleBucket(d1, d2); + assertBucketRange(expected, new DoubleValue(d1next), false, new DoubleValue(d2next), true); + assertBucketRange(expected, new DoubleValue(d1next), false, new DoubleValue(d2), false); + assertBucketRange(expected, new DoubleValue(d1), true, new DoubleValue(d2next), true); + assertBucketRange(expected, new DoubleValue(d1), true, new DoubleValue(d2), false); + + resolver = new BucketResolver(); + resolver.push(new DoubleValue(d1next), true).push(new DoubleValue(d1), false); + resolver.push(new DoubleValue(d1), true).push(new DoubleValue(d2next), true); + resolver.push(new DoubleValue(d2next), false).push(new DoubleValue(d2), false); + resolver.push(new DoubleValue(d2), false).push(new DoubleValue(ChoiceFormat.nextDouble(d2)), true); + assertBucketOrder(resolver); + } + + // -------------------------------------------------------------------------------- + // + // Utilities + // + // -------------------------------------------------------------------------------- + + private static void checkPushFail(List<ConstantValue> args, String expectedException) { + BucketResolver resolver = new BucketResolver(); + try { + int i = 0; + for (ConstantValue exp : args) { + boolean inclusive = ((i % 2) == 0); + resolver.push(exp, inclusive); + i++; + } + fail(); + } catch (IllegalArgumentException e) { + assertEquals(expectedException, e.getMessage()); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/ExpressionVisitorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/ExpressionVisitorTestCase.java new file mode 100644 index 00000000000..f5d30497671 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/ExpressionVisitorTestCase.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ExpressionVisitorTestCase { + + @Test + public void requireThatExpressionsAreVisited() { + GroupingOperation op = new AllOperation(); + + final List<GroupingExpression> lst = new LinkedList<>(); + GroupingExpression exp = new AttributeValue("groupBy"); + op.setGroupBy(exp); + lst.add(exp); + + op.addOrderBy(exp = new AttributeValue("orderBy1")); + lst.add(exp); + op.addOrderBy(exp = new AttributeValue("orderBy1")); + lst.add(exp); + + op.addOutput(exp = new AttributeValue("output1")); + lst.add(exp); + op.addOutput(exp = new AttributeValue("output2")); + lst.add(exp); + + op.visitExpressions(exp1 -> assertNotNull(lst.remove(exp1))); + assertTrue(lst.isEmpty()); + } + + @Test + public void requireThatChildOperationsAreVisited() { + GroupingOperation root, parentA, childA1, childA2, parentB, childB1; + root = new AllOperation() + .addChild(parentA = new AllOperation() + .addChild(childA1 = new AllOperation()) + .addChild(childA2 = new AllOperation())) + .addChild(parentB = new AllOperation() + .addChild(childB1 = new AllOperation())); + + final List<GroupingExpression> lst = new LinkedList<>(); + GroupingExpression exp = new AttributeValue("parentA"); + parentA.setGroupBy(exp); + lst.add(exp); + + childA1.setGroupBy(exp = new AttributeValue("childA1")); + lst.add(exp); + + childA2.setGroupBy(exp = new AttributeValue("childA2")); + lst.add(exp); + + parentB.setGroupBy(exp = new AttributeValue("parentB")); + lst.add(exp); + + childB1.setGroupBy(exp = new AttributeValue("childB1")); + lst.add(exp); + + root.visitExpressions(exp1 -> assertNotNull(lst.remove(exp1))); + assertTrue(lst.isEmpty()); + } + + @Test + public void requireThatExpressionsArgumentsAreVisited() { + final List<GroupingExpression> lst = new LinkedList<>(); + GroupingExpression arg1 = new AttributeValue("arg1"); + lst.add(arg1); + GroupingExpression arg2 = new AttributeValue("arg2"); + lst.add(arg2); + + new AndFunction(arg1, arg2).visit(exp -> assertNotNull(lst.remove(exp))); + assertTrue(lst.isEmpty()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/GroupingOperationTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/GroupingOperationTestCase.java new file mode 100644 index 00000000000..614a126b54d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/GroupingOperationTestCase.java @@ -0,0 +1,148 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import com.yahoo.search.grouping.request.parser.ParseException; +import com.yahoo.search.grouping.request.parser.TokenMgrError; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupingOperationTestCase { + + @Test + public void requireThatAccessorsWork() { + GroupingOperation op = new AllOperation(); + GroupingExpression exp = new AttributeValue("alias"); + op.putAlias("alias", exp); + assertSame(exp, op.getAlias("alias")); + + assertEquals(0, op.getHints().size()); + assertFalse(op.containsHint("foo")); + assertFalse(op.containsHint("bar")); + + op.addHint("foo"); + assertEquals(1, op.getHints().size()); + assertTrue(op.containsHint("foo")); + assertFalse(op.containsHint("bar")); + + op.addHint("bar"); + assertEquals(2, op.getHints().size()); + assertTrue(op.containsHint("foo")); + assertTrue(op.containsHint("bar")); + + op.setForceSinglePass(true); + assertTrue(op.getForceSinglePass()); + op.setForceSinglePass(false); + assertFalse(op.getForceSinglePass()); + + exp = new AttributeValue("orderBy"); + op.addOrderBy(exp); + assertEquals(1, op.getOrderBy().size()); + assertSame(exp, op.getOrderBy(0)); + + exp = new AttributeValue("output"); + op.addOutput(exp); + assertEquals(1, op.getOutputs().size()); + assertSame(exp, op.getOutput(0)); + + GroupingOperation child = new AllOperation(); + op.addChild(child); + assertEquals(1, op.getChildren().size()); + assertSame(child, op.getChild(0)); + + exp = new AttributeValue("groupBy"); + op.setGroupBy(exp); + assertSame(exp, op.getGroupBy()); + + op.setWhere("whereA"); + assertEquals("whereA", op.getWhere()); + op.setWhere("whereB"); + assertEquals("whereB", op.getWhere()); + + op.setAccuracy(0.6); + assertEquals(0.6, op.getAccuracy(), 1E-6); + op.setAccuracy(0.9); + assertEquals(0.9, op.getAccuracy(), 1E-6); + + op.setPrecision(6); + assertEquals(6, op.getPrecision()); + op.setPrecision(9); + assertEquals(9, op.getPrecision()); + + assertFalse(op.hasMax()); + op.setMax(6); + assertTrue(op.hasMax()); + assertEquals(6, op.getMax()); + op.setMax(9); + assertEquals(9, op.getMax()); + assertTrue(op.hasMax()); + op.setMax(0); + assertTrue(op.hasMax()); + op.setMax(-7); + assertFalse(op.hasMax()); + } + + @Test + public void requireThatFromStringAsListParsesAllOperations() { + List<GroupingOperation> lst = GroupingOperation.fromStringAsList(""); + assertTrue(lst.isEmpty()); + + lst = GroupingOperation.fromStringAsList("all()"); + assertEquals(1, lst.size()); + assertTrue(lst.get(0) instanceof AllOperation); + + lst = GroupingOperation.fromStringAsList("each()"); + assertEquals(1, lst.size()); + assertTrue(lst.get(0) instanceof EachOperation); + + lst = GroupingOperation.fromStringAsList("all();each()"); + assertEquals(2, lst.size()); + assertTrue(lst.get(0) instanceof AllOperation); + assertTrue(lst.get(1) instanceof EachOperation); + } + + @Test + public void requireThatFromStringAcceptsOnlyOneOperation() { + try { + GroupingOperation.fromString(""); + fail(); + } catch (IllegalArgumentException e) { + + } + assertTrue(GroupingOperation.fromString("all()") instanceof AllOperation); + assertTrue(GroupingOperation.fromString("each()") instanceof EachOperation); + try { + GroupingOperation.fromString("all();each()"); + fail(); + } catch (IllegalArgumentException e) { + + } + } + + @Test + public void requireThatParseExceptionsAreRethrown() { + try { + GroupingOperation.fromString("all(foo)"); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("Encountered \"foo\" at line 1, column 5.\n")); + assertTrue(e.getCause() instanceof ParseException); + } + } + + @Test + public void requireThatTokenErrorsAreRethrown() { + try { + GroupingOperation.fromString("all(\\foo)"); + fail(); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().startsWith("Lexical error at line 1, column 6.")); + assertTrue(e.getCause() instanceof TokenMgrError); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java new file mode 100644 index 00000000000..14274e98182 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/MathFunctionsTestCase.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertThat; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.9 + */ +public class MathFunctionsTestCase { + @Test + public void testMathFunctions() { + //if this fails, update the count AND add a test in each of the two blocks below + assertThat(MathFunctions.Function.values().length, is(21)); + + + assertThat(MathFunctions.Function.create(0), sameInstance(MathFunctions.Function.EXP)); + assertThat(MathFunctions.Function.create(1), sameInstance(MathFunctions.Function.POW)); + assertThat(MathFunctions.Function.create(2), sameInstance(MathFunctions.Function.LOG)); + assertThat(MathFunctions.Function.create(3), sameInstance(MathFunctions.Function.LOG1P)); + assertThat(MathFunctions.Function.create(4), sameInstance(MathFunctions.Function.LOG10)); + assertThat(MathFunctions.Function.create(5), sameInstance(MathFunctions.Function.SIN)); + assertThat(MathFunctions.Function.create(6), sameInstance(MathFunctions.Function.ASIN)); + assertThat(MathFunctions.Function.create(7), sameInstance(MathFunctions.Function.COS)); + assertThat(MathFunctions.Function.create(8), sameInstance(MathFunctions.Function.ACOS)); + assertThat(MathFunctions.Function.create(9), sameInstance(MathFunctions.Function.TAN)); + assertThat(MathFunctions.Function.create(10), sameInstance(MathFunctions.Function.ATAN)); + assertThat(MathFunctions.Function.create(11), sameInstance(MathFunctions.Function.SQRT)); + assertThat(MathFunctions.Function.create(12), sameInstance(MathFunctions.Function.SINH)); + assertThat(MathFunctions.Function.create(13), sameInstance(MathFunctions.Function.ASINH)); + assertThat(MathFunctions.Function.create(14), sameInstance(MathFunctions.Function.COSH)); + assertThat(MathFunctions.Function.create(15), sameInstance(MathFunctions.Function.ACOSH)); + assertThat(MathFunctions.Function.create(16), sameInstance(MathFunctions.Function.TANH)); + assertThat(MathFunctions.Function.create(17), sameInstance(MathFunctions.Function.ATANH)); + assertThat(MathFunctions.Function.create(18), sameInstance(MathFunctions.Function.CBRT)); + assertThat(MathFunctions.Function.create(19), sameInstance(MathFunctions.Function.HYPOT)); + assertThat(MathFunctions.Function.create(20), sameInstance(MathFunctions.Function.FLOOR)); + + + assertThat(MathFunctions.newInstance(MathFunctions.Function.EXP, null, null), instanceOf(MathExpFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.POW, null, null), instanceOf(MathPowFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.LOG, null, null), instanceOf(MathLogFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.LOG1P, null, null), instanceOf(MathLog1pFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.LOG10, null, null), instanceOf(MathLog10Function.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.SIN, null, null), instanceOf(MathSinFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ASIN, null, null), instanceOf(MathASinFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.COS, null, null), instanceOf(MathCosFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ACOS, null, null), instanceOf(MathACosFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.TAN, null, null), instanceOf(MathTanFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ATAN, null, null), instanceOf(MathATanFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.SQRT, null, null), instanceOf(MathSqrtFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.SINH, null, null), instanceOf(MathSinHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ASINH, null, null), instanceOf(MathASinHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.COSH, null, null), instanceOf(MathCosHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ACOSH, null, null), instanceOf(MathACosHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.TANH, null, null), instanceOf(MathTanHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.ATANH, null, null), instanceOf(MathATanHFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.CBRT, null, null), instanceOf(MathCbrtFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.HYPOT, null, null), instanceOf(MathHypotFunction.class)); + assertThat(MathFunctions.newInstance(MathFunctions.Function.FLOOR, null, null), instanceOf(MathFloorFunction.class)); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/MathResolverTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/MathResolverTestCase.java new file mode 100644 index 00000000000..af007a6f85c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/MathResolverTestCase.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class MathResolverTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests + // + // -------------------------------------------------------------------------------- + + @Test + public void testOperators() { + MathResolver resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.ADD, new LongValue(2)); + assertEquals("add(1, 2)", + resolver.resolve().toString()); + + resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.SUB, new LongValue(2)); + assertEquals("sub(1, 2)", + resolver.resolve().toString()); + + resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.DIV, new LongValue(2)); + assertEquals("div(1, 2)", + resolver.resolve().toString()); + + resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.MOD, new LongValue(2)); + assertEquals("mod(1, 2)", + resolver.resolve().toString()); + + resolver = new MathResolver(); + resolver.push(MathResolver.Type.ADD, new LongValue(1)); + resolver.push(MathResolver.Type.MUL, new LongValue(2)); + assertEquals("mul(1, 2)", + resolver.resolve().toString()); + } + + @Test + public void testOperatorPrecedence() { + assertResolve("add(add(1, 2), 3)", MathResolver.Type.ADD, MathResolver.Type.ADD); + assertResolve("add(1, sub(2, 3))", MathResolver.Type.ADD, MathResolver.Type.SUB); + assertResolve("add(1, div(2, 3))", MathResolver.Type.ADD, MathResolver.Type.DIV); + assertResolve("add(1, mod(2, 3))", MathResolver.Type.ADD, MathResolver.Type.MOD); + assertResolve("add(1, mul(2, 3))", MathResolver.Type.ADD, MathResolver.Type.MUL); + + assertResolve("add(sub(1, 2), 3)", MathResolver.Type.SUB, MathResolver.Type.ADD); + assertResolve("sub(sub(1, 2), 3)", MathResolver.Type.SUB, MathResolver.Type.SUB); + assertResolve("sub(1, div(2, 3))", MathResolver.Type.SUB, MathResolver.Type.DIV); + assertResolve("sub(1, mod(2, 3))", MathResolver.Type.SUB, MathResolver.Type.MOD); + assertResolve("sub(1, mul(2, 3))", MathResolver.Type.SUB, MathResolver.Type.MUL); + + assertResolve("add(div(1, 2), 3)", MathResolver.Type.DIV, MathResolver.Type.ADD); + assertResolve("sub(div(1, 2), 3)", MathResolver.Type.DIV, MathResolver.Type.SUB); + assertResolve("div(div(1, 2), 3)", MathResolver.Type.DIV, MathResolver.Type.DIV); + assertResolve("div(1, mod(2, 3))", MathResolver.Type.DIV, MathResolver.Type.MOD); + assertResolve("div(1, mul(2, 3))", MathResolver.Type.DIV, MathResolver.Type.MUL); + + assertResolve("add(mod(1, 2), 3)", MathResolver.Type.MOD, MathResolver.Type.ADD); + assertResolve("sub(mod(1, 2), 3)", MathResolver.Type.MOD, MathResolver.Type.SUB); + assertResolve("div(mod(1, 2), 3)", MathResolver.Type.MOD, MathResolver.Type.DIV); + assertResolve("mod(mod(1, 2), 3)", MathResolver.Type.MOD, MathResolver.Type.MOD); + assertResolve("mod(1, mul(2, 3))", MathResolver.Type.MOD, MathResolver.Type.MUL); + + assertResolve("add(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.ADD); + assertResolve("sub(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.SUB); + assertResolve("div(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.DIV); + assertResolve("mod(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.MOD); + assertResolve("mul(mul(1, 2), 3)", MathResolver.Type.MUL, MathResolver.Type.MUL); + + assertResolve("add(1, sub(div(2, mod(3, mul(4, 5))), 6))", + MathResolver.Type.ADD, MathResolver.Type.DIV, MathResolver.Type.MOD, + MathResolver.Type.MUL, MathResolver.Type.SUB); + assertResolve("add(sub(1, div(mod(mul(2, 3), 4), 5)), 6)", + MathResolver.Type.SUB, MathResolver.Type.MUL, MathResolver.Type.MOD, + MathResolver.Type.DIV, MathResolver.Type.ADD); + assertResolve("add(1, sub(2, div(3, mod(4, mul(5, 6)))))", + MathResolver.Type.ADD, MathResolver.Type.SUB, MathResolver.Type.DIV, + MathResolver.Type.MOD, MathResolver.Type.MUL); + assertResolve("add(sub(div(mod(mul(1, 2), 3), 4), 5), 6)", + MathResolver.Type.MUL, MathResolver.Type.MOD, MathResolver.Type.DIV, + MathResolver.Type.SUB, MathResolver.Type.ADD); + } + + @Test + public void testOperatorSupport() { + MathResolver resolver = new MathResolver(); + for (MathResolver.Type type : MathResolver.Type.values()) { + if (type == MathResolver.Type.ADD) { + continue; + } + try { + resolver.push(type, new AttributeValue("foo")); + } catch (IllegalArgumentException e) { + assertEquals("First item in an arithmetic operation must be an addition.", e.getMessage()); + } + } + } + + // -------------------------------------------------------------------------------- + // + // Utilities + // + // -------------------------------------------------------------------------------- + + private static void assertResolve(String expected, MathResolver.Type... types) { + MathResolver resolver = new MathResolver(); + + int val = 0; + resolver.push(MathResolver.Type.ADD, new LongValue(++val)); + for (MathResolver.Type type : types) { + resolver.push(type, new LongValue(++val)); + } + + GroupingExpression exp = resolver.resolve(); + assertNotNull(exp); + assertEquals(expected, exp.toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java new file mode 100644 index 00000000000..eba5a458cfd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/RawBufferTestCase.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + */ +public class RawBufferTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests. + // + // -------------------------------------------------------------------------------- + + @Test + public void requireThatCompareWorks() { + RawBuffer buffer = new RawBuffer(); + buffer.put((byte)'a'); + buffer.put((byte)'b'); + + RawBuffer buffer2 = new RawBuffer(); + buffer2.put((byte)'k'); + buffer2.put((byte)'a'); + + ArrayList<Byte> backing = new ArrayList<>(); + backing.add((byte)'a'); + backing.add((byte)'b'); + RawBuffer buffer3 = new RawBuffer(backing); + + assertEquals(buffer.compareTo(buffer2), -1); + assertEquals(buffer2.compareTo(buffer), 1); + assertEquals(buffer.compareTo(buffer3), 0); + } + + @Test + public void requireThatToStringWorks() { + assertToString(Arrays.asList("a".getBytes()[0], "b".getBytes()[0]), "{97,98}"); + assertToString(Arrays.asList(new Byte((byte)2), new Byte((byte)6)), "{2,6}"); + } + + public void assertToString(List<Byte> data, String expected) { + RawBuffer buffer = new RawBuffer(); + for (Byte b : data) { + buffer.put(b.byteValue()); + } + assertEquals(buffer.toString(), expected); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java new file mode 100644 index 00000000000..f2f8316f2db --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/RequestTestCase.java @@ -0,0 +1,229 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request; + +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RequestTestCase { + + @Test + public void requireThatApiWorks() { + GroupingOperation op = new AllOperation() + .setGroupBy(new AttributeValue("foo")) + .addOrderBy(new CountAggregator()) + .addChildren(Arrays.asList(new AllOperation(), new EachOperation())) + .addChild(new EachOperation() + .addOutput(new CountAggregator()) + .addOutput(new MinAggregator(new AttributeValue("bar"))) + .addChild(new EachOperation() + .addOutput(new AddFunction( + new LongValue(69), + new AttributeValue("baz"))) + .addOutput(new SummaryValue("cox")))); + assertEquals("all(group(foo) order(count()) all() each() " + + "each(output(count(), min(bar)) each(output(add(69, baz), summary(cox)))))", + op.toString()); + op.resolveLevel(1); + + GroupingExpression exp = op.getGroupBy(); + assertNotNull(exp); + assertTrue(exp instanceof AttributeValue); + assertEquals("foo", ((AttributeValue)exp).getAttributeName()); + assertEquals(1, op.getNumOrderBy()); + assertNotNull(exp = op.getOrderBy(0)); + assertTrue(exp instanceof CountAggregator); + + assertEquals(3, op.getNumChildren()); + assertTrue(op.getChild(0) instanceof AllOperation); + assertTrue(op.getChild(1) instanceof EachOperation); + assertNotNull(op = op.getChild(2)); + assertTrue(op instanceof EachOperation); + assertEquals(2, op.getNumOutputs()); + assertNotNull(exp = op.getOutput(0)); + assertTrue(exp instanceof CountAggregator); + assertNotNull(exp = op.getOutput(1)); + assertTrue(exp instanceof MinAggregator); + assertNotNull(exp = ((MinAggregator)exp).getExpression()); + assertTrue(exp instanceof AttributeValue); + assertEquals("bar", ((AttributeValue)exp).getAttributeName()); + + assertEquals(1, op.getNumChildren()); + assertNotNull(op = op.getChild(0)); + assertTrue(op instanceof EachOperation); + assertEquals(2, op.getNumOutputs()); + assertNotNull(exp = op.getOutput(0)); + assertTrue(exp instanceof AddFunction); + assertEquals(2, ((AddFunction)exp).getNumArgs()); + GroupingExpression arg = ((AddFunction)exp).getArg(0); + assertNotNull(arg); + assertTrue(arg instanceof LongValue); + assertEquals(69L, ((LongValue)arg).getValue().longValue()); + assertNotNull(arg = ((AddFunction)exp).getArg(1)); + assertTrue(arg instanceof AttributeValue); + assertEquals("baz", ((AttributeValue)arg).getAttributeName()); + assertNotNull(exp = op.getOutput(1)); + assertTrue(exp instanceof SummaryValue); + assertEquals("cox", ((SummaryValue)exp).getSummaryName()); + } + + @Test + public void requireThatPredefinedApiWorks() { + PredefinedFunction fnc = new LongPredefined(new AttributeValue("foo"), + new LongBucket(1, 2), + new LongBucket(3, 4)); + assertEquals(2, fnc.getNumBuckets()); + BucketValue bucket = fnc.getBucket(0); + assertNotNull(bucket); + assertTrue(bucket instanceof LongBucket); + assertEquals(1L, bucket.getFrom().getValue()); + assertEquals(2L, bucket.getTo().getValue()); + + assertNotNull(bucket = fnc.getBucket(1)); + assertTrue(bucket instanceof LongBucket); + assertEquals(3L, bucket.getFrom().getValue()); + assertEquals(4L, bucket.getTo().getValue()); + } + + @Test + public void requireThatBucketIntegrityIsChecked() { + try { + new LongBucket(2, 1); + } catch (IllegalArgumentException e) { + assertEquals("Bucket to-value can not be less than from-value.", e.getMessage()); + } + try { + new LongPredefined(new AttributeValue("foo"), + new LongBucket(3, 4), + new LongBucket(1, 2)); + } catch (IllegalArgumentException e) { + assertEquals("Buckets must be monotonically increasing, got bucket[3, 4> before bucket[1, 2>.", + e.getMessage()); + } + } + + @Test + public void requireThatAliasWorks() { + GroupingOperation all = new AllOperation(); + all.putAlias("myalias", new AttributeValue("foo")); + GroupingExpression exp = all.getAlias("myalias"); + assertNotNull(exp); + assertTrue(exp instanceof AttributeValue); + assertEquals("foo", ((AttributeValue)exp).getAttributeName()); + + GroupingOperation each = new EachOperation(); + all.addChild(each); + assertNotNull(exp = each.getAlias("myalias")); + assertTrue(exp instanceof AttributeValue); + assertEquals("foo", ((AttributeValue)exp).getAttributeName()); + + each.putAlias("myalias", new AttributeValue("bar")); + assertNotNull(exp = each.getAlias("myalias")); + assertTrue(exp instanceof AttributeValue); + assertEquals("bar", ((AttributeValue)exp).getAttributeName()); + } + + @Test + public void testOrderBy() { + GroupingOperation all = new AllOperation(); + all.addOrderBy(new AttributeValue("foo")); + try { + all.resolveLevel(0); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Operation 'all(order(foo))' can not order single hit.", e.getMessage()); + } + all.resolveLevel(1); + assertEquals(0, all.getOrderBy(0).getLevel()); + } + + @Test + public void testMax() { + GroupingOperation all = new AllOperation(); + all.setMax(69); + try { + all.resolveLevel(0); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Operation 'all(max(69))' can not apply max to single hit.", e.getMessage()); + } + all.resolveLevel(1); + } + + @Test + public void testAccuracy() { + GroupingOperation all = new AllOperation(); + all.setAccuracy(0.53); + assertEquals((long)(100.0 * all.getAccuracy()), 53); + try { + all.setAccuracy(1.2); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Illegal accuracy '1.2'. Must be between 0 and 1.", e.getMessage()); + } + try { + all.setAccuracy(-0.5); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Illegal accuracy '-0.5'. Must be between 0 and 1.", e.getMessage()); + } + } + + @Test + public void testLevelChange() { + GroupingOperation all = new AllOperation(); + all.resolveLevel(0); + assertEquals(0, all.getLevel()); + all.setGroupBy(new AttributeValue("foo")); + all.resolveLevel(1); + assertEquals(2, all.getLevel()); + + GroupingOperation each = new EachOperation(); + try { + each.resolveLevel(0); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Operation '" + each + "' can not operate on single hit.", e.getMessage()); + } + each.resolveLevel(1); + assertEquals(0, each.getLevel()); + each.setGroupBy(new AttributeValue("foo")); + each.resolveLevel(2); + assertEquals(2, each.getLevel()); + } + + @Test + public void testLevelInheritance() { + GroupingOperation grandParent, parent, child, grandChild; + grandParent = new AllOperation() + .addChild(parent = new EachOperation() + .addChild(child = new AllOperation() + .addChild(grandChild = new EachOperation()))); + + grandParent.resolveLevel(69); + assertEquals(69, grandParent.getLevel()); + assertEquals(68, parent.getLevel()); + assertEquals(68, child.getLevel()); + assertEquals(67, grandChild.getLevel()); + } + + @Test + public void testLevelPropagation() { + GroupingOperation all = new AllOperation() + .setGroupBy(new AttributeValue("foo")) + .addOrderBy(new MaxAggregator(new AttributeValue("bar"))) + .addChild(new EachOperation() + .addOutput(new MaxAggregator(new AttributeValue("baz")))); + + all.resolveLevel(1); + assertEquals(0, all.getGroupBy().getLevel()); + assertEquals(1, all.getOrderBy(0).getLevel()); + assertEquals(1, all.getChild(0).getOutput(0).getLevel()); + assertEquals(0, ((AggregatorNode)all.getChild(0).getOutput(0)).getExpression().getLevel()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java new file mode 100644 index 00000000000..2abd4a01dd7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserBenchmarkTest.java @@ -0,0 +1,270 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request.parser; + +import com.yahoo.search.grouping.request.GroupingOperation; +import org.junit.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupingParserBenchmarkTest { + + private static final int NUM_RUNS = 10;//000; + private static final Map<String, Long> PREV_RESULTS = new LinkedHashMap<>(); + + static { + PREV_RESULTS.put("Original", 79276393L); + PREV_RESULTS.put("NoCache", 71728602L); + PREV_RESULTS.put("CharStream", 43852348L); + PREV_RESULTS.put("CharArray", 22936513L); + } + + @Test + public void requireThatGroupingParserIsFast() { + List<String> inputs = getInputs(); + long ignore = 0; + long now = 0; + for (int i = 0; i < 2; ++i) { + now = System.nanoTime(); + for (int j = 0; j < NUM_RUNS; ++j) { + for (String str : inputs) { + ignore += parseRequest(str); + } + } + } + long micros = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - now); + System.out.format("%d \u03bcs (avg %.2f)\n", micros, (double)micros / (NUM_RUNS * inputs.size())); + for (Map.Entry<String, Long> entry : PREV_RESULTS.entrySet()) { + System.out.format("%-20s : %4.2f\n", entry.getKey(), (double)micros / entry.getValue()); + } + System.out.println("\nignore " + ignore); + } + + private static int parseRequest(String str) { + return GroupingOperation.fromStringAsList(str).size(); + } + + private static List<String> getInputs() { + return Arrays.asList( + " all(group(foo)each(output(max(bar))))", + "all( group(foo)each(output(max(bar))))", + "all(group( foo)each(output(max(bar))))", + "all(group(foo )each(output(max(bar))))", + "all(group(foo) each(output(max(bar))))", + "all(group(foo)each( output(max(bar))))", + "all(group(foo)each(output( max(bar))))", + "all(group(foo)each(output(max(bar))))", + "all(group(foo)each(output(max( bar))))", + "all(group(foo)each(output(max(bar ))))", + "all(group(foo)each(output(max(bar) )))", + "all(group(foo)each(output(max(bar)) ))", + "all(group(foo)each(output(max(bar))) )", + "all(group(foo)each(output(max(bar)))) ", + "all()", + "each()", + "all(each())", + "each(all())", + "all(all() all())", + "all(all() each())", + "all(each() all())", + "all(each() each())", + "each(all() all())", + "all(group(foo))", + "all(max(1))", + "all(order(foo))", + "all(order(+foo))", + "all(order(-foo))", + "all(order(foo, bar))", + "all(order(foo, +bar))", + "all(order(foo, -bar))", + "all(order(+foo, bar))", + "all(order(+foo, +bar))", + "all(order(+foo, -bar))", + "all(order(-foo, bar))", + "all(order(-foo, +bar))", + "all(order(-foo, -bar))", + "all(output(min(a)))", + "all(output(min(a), min(b)))", + "all(precision(1))", + "all(where(foo))", + "all(where($foo))", + "all(group(fixedwidth(foo, 1)))", + "all(group(fixedwidth(foo, 1.2)))", + "all(group(md5(foo, 1)))", + "all(group(predefined(foo, bucket(1, 2))))", + "all(group(predefined(foo, bucket(-1, 2))))", + "all(group(predefined(foo, bucket(-2, -1))))", + "all(group(predefined(foo, bucket(1, 2), bucket(3, 4))))", + "all(group(predefined(foo, bucket(1, 2), bucket(3, 4), bucket(5, 6))))", + "all(group(predefined(foo, bucket(1, 2), bucket(2, 3), bucket(3, 4))))", + "all(group(predefined(foo, bucket(-100, 0), bucket(0), bucket<0, 100))))", + "all(group(predefined(foo, bucket[1, 2>)))", + "all(group(predefined(foo, bucket[-1, 2>)))", + "all(group(predefined(foo, bucket[-2, -1>)))", + "all(group(predefined(foo, bucket[1, 2>, bucket(3, 4>)))", + "all(group(predefined(foo, bucket[1, 2>, bucket[3, 4>, bucket[5, 6>)))", + "all(group(predefined(foo, bucket[1, 2>, bucket[2, 3>, bucket[3, 4>)))", + "all(group(predefined(foo, bucket<1, 5>)))", + "all(group(predefined(foo, bucket[1, 5>)))", + "all(group(predefined(foo, bucket<1, 5])))", + "all(group(predefined(foo, bucket[1, 5])))", + "all(group(predefined(foo, bucket<1, inf>)))", + "all(group(predefined(foo, bucket<-inf, -1>)))", + "all(group(predefined(foo, bucket<a, inf>)))", + "all(group(predefined(foo, bucket<'a', inf>)))", + "all(group(predefined(foo, bucket<-inf, a>)))", + "all(group(predefined(foo, bucket[-inf, 'a'>)))", + "all(group(predefined(foo, bucket<-inf, -0.3>)))", + "all(group(predefined(foo, bucket<0.3, inf])))", + "all(group(predefined(foo, bucket<0.3, inf])))", + "all(group(predefined(foo, bucket<infinite, inf])))", + "all(group(predefined(foo, bucket<myinf, inf])))", + "all(group(predefined(foo, bucket<-inf, infinite])))", + "all(group(predefined(foo, bucket<-inf, myinf])))", + "all(group(predefined(foo, bucket(1.0, 2.0))))", + "all(group(predefined(foo, bucket(1.0, 2.0), bucket(3.0, 4.0))))", + "all(group(predefined(foo, bucket(1.0, 2.0), bucket(3.0, 4.0), bucket(5.0, 6.0))))", + "all(group(predefined(foo, bucket<1.0, 2.0>)))", + "all(group(predefined(foo, bucket[1.0, 2.0>)))", + "all(group(predefined(foo, bucket<1.0, 2.0])))", + "all(group(predefined(foo, bucket[1.0, 2.0])))", + "all(group(predefined(foo, bucket[1.0, 2.0>, bucket[3.0, 4.0>)))", + "all(group(predefined(foo, bucket[1.0, 2.0>, bucket[3.0, 4.0>, bucket[5.0, 6.0>)))", + "all(group(predefined(foo, bucket[1.0, 2.0>, bucket[2.0], bucket<2.0, 6.0>)))", + "all(group(predefined(foo, bucket('a', 'b'))))", + "all(group(predefined(foo, bucket['a', 'b'>)))", + "all(group(predefined(foo, bucket<'a', 'c'>)))", + "all(group(predefined(foo, bucket<'a', 'b'])))", + "all(group(predefined(foo, bucket['a', 'b'])))", + "all(group(predefined(foo, bucket('a', 'b'), bucket('c', 'd'))))", + "all(group(predefined(foo, bucket('a', 'b'), bucket('c', 'd'), bucket('e', 'f'))))", + "all(group(predefined(foo, bucket(\"a\", \"b\"))))", + "all(group(predefined(foo, bucket(\"a\", \"b\"), bucket(\"c\", \"d\"))))", + "all(group(predefined(foo, bucket(\"a\", \"b\"), bucket(\"c\", \"d\"), bucket(\"e\", \"f\"))))", + "all(group(predefined(foo, bucket(a, b))))", + "all(group(predefined(foo, bucket(a, b), bucket(c, d))))", + "all(group(predefined(foo, bucket(a, b), bucket(c, d), bucket(e, f))))", + "all(group(predefined(foo, bucket(a, b), bucket(c), bucket(e, f))))", + "all(group(predefined(foo, bucket('a', \"b\"))))", + "all(group(predefined(foo, bucket('a', \"b\"), bucket(c, 'd'))))", + "all(group(predefined(foo, bucket('a', \"b\"), bucket(c, 'd'), bucket(\"e\", f))))", + "all(group(predefined(foo, bucket('a(', \"b)\"), bucket(c, 'd()'))))", + "all(group(predefined(foo, bucket({2}, {6}), bucket({7}, {12}))))", + "all(group(predefined(foo, bucket({0, 2}, {0, 6}), bucket({0, 7}, {0, 12}))))", + "all(group(predefined(foo, bucket({'b', 'a'}, {'k', 'a'}), bucket({'k', 'a'}, {'u', 'b'}))))", + "all(group(xorbit(foo, 1)))", + "all(group(1))", + "all(group(1+2))", + "all(group(1-2))", + "all(group(1*2))", + "all(group(1/2))", + "all(group(1%2))", + "all(group(1+2+3))", + "all(group(1+2-3))", + "all(group(1+2*3))", + "all(group(1+2/3))", + "all(group(1+2%3))", + "all(group((1+2)+3))", + "all(group((1+2)-3))", + "all(group((1+2)*3))", + "all(group((1+2)/3))", + "all(group((1+2)%3))", + "each() as(foo)", + "all(each() as(foo) each() as(bar))", + "all(group(a) each(each() as(foo) each() as(bar)) each() as(baz))", + "all(output(min(a) as(foo)))", + "all(output(min(a) as(foo), max(b) as(bar)))", + "all(where(bar) all(group(foo)))", + "all(group(foo)) where(bar)", + "all(group(attribute(foo)))", + "all(group(attribute(foo)) order(sum(attribute(a))))", + "all(accuracy(0.5))", + "all(group(foo) accuracy(1.0))", + "all(group(my.little{key}))", "all(group(my.little{\"key\"}))", + "all(group(my.little{key }))", "all(group(my.little{\"key\"}))", + "all(group(my.little{\"key\"}))", "all(group(my.little{\"key\"}))", + "all(group(my.little{\"key{}%\"}))", "all(group(my.little{\"key{}%\"}))", + "all(group(artist) max(7))", + "all(max(76) all(group(artist) max(7)))", + "all(group(artist) max(7) where(true))", + "all(group(artist) order(sum(a)) output(count()))", + "all(group(artist) order(+sum(a)) output(count()))", + "all(group(artist) order(-sum(a)) output(count()))", + "all(group(artist) order(-sum(a), +xor(b)) output(count()))", + "all(group(artist) max(1) output(count()))", + "all(group(-(m)) max(1) output(count()))", + "all(group(min) max(1) output(count()))", + "all(group(artist) max(2) each(each(output(summary()))))", + "all(group(artist) max(2) each(each(output(summary(simple)))))", + "all(group(artist) max(5) each(output(count()) each(output(summary()))))", + "all(group(ymum()))", + "all(group(strlen(attr)))", + "all(group(normalizesubject(attr)))", + "all(group(strcat(attr, attr2)))", + "all(group(tostring(attr)))", + "all(group(toraw(attr)))", + "all(group(zcurve.x(attr)))", + "all(group(zcurve.y(attr)))", + "all(group(uca(attr, \"foo\")))", + "all(group(uca(attr, \"foo\", \"PRIMARY\")))", + "all(group(uca(attr, \"foo\", \"SECONDARY\")))", + "all(group(uca(attr, \"foo\", \"TERTIARY\")))", + "all(group(uca(attr, \"foo\", \"QUATERNARY\")))", + "all(group(uca(attr, \"foo\", \"IDENTICAL\")))", + "all(group(tolong(attr)))", + "all(group(sort(attr)))", + "all(group(reverse(attr)))", + "all(group(docidnsspecific()))", + "all(group(relevance()))", + "all(group(artist) each(each(output(summary()))))", + "all(group(artist) max(13) " + + " each(group(fixedwidth(year, 21.34)) max(55) output(count()) " + + " each(each(output(summary())))))", + "all(group(artist) max(13) " + + " each(group(predefined(year, bucket(7, 19), bucket(90, 300))) max(55) output(count()) " + + " each(each(output(summary())))))", + "all(group(artist) max(13) " + + " each(group(predefined(year, bucket(7.1, 19.0), bucket(90.7, 300.0))) max(55) output(count()) " + + " each(each(output(summary())))))", + "all(group(artist) max(13) " + + " each(group(predefined(year, bucket('a', 'b'), bucket('peder', 'pedersen'))) " + + " max(55) output(count()) " + + " each(each(output(summary())))))", + "all(output(count()))", + "all(group(album) output(count()))", + "all(group(album) each(output(count())))", + "all(group(artist) each(group(album) output(count()))" + + " each(group(song) output(count())))", + "all(group(artist) output(count())" + + " each(group(album) output(count())" + + " each(group(song) output(count())" + + " each(each(output(summary()))))))", + "all(group(album) order(-$total=sum(length)) each(output($total)))", + "all(group(album) max(1) each(output(sum(length))))", + "all(group(artist) each(max(2) each(output(summary()))))", + "all(group(artist) max(3)" + + " each(group(album as(albumsongs)) each(each(output(summary()))))" + + " each(group(album as(albumlength)) output(sum(sum(length)))))", + "all(group(artist) max(15)" + + " each(group(album) " + + " each(group(song)" + + " each(max(2) each(output(summary()))))))", + "all(group(artist) max(15)" + + " each(group(album)" + + " each(group(song)" + + " each(max(2) each(output(summary())))))" + + " each(group(song) max(5) order(sum(popularity))" + + " each(output(sum(sold)) each(output(summary())))))", + "all(group(artist) order(max(relevance) * count()) each(output(count())))", + "all(group(artist) each(output(sum(popularity) / count())))", + "all(group(artist) accuracy(0.1) each(output(sum(popularity) / count())))", + "all(group(debugwait(artist, 3.3, true)))", + "all(group(debugwait(artist, 3.3, false)))"); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java new file mode 100644 index 00000000000..c9fbcad28f2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/request/parser/GroupingParserTestCase.java @@ -0,0 +1,619 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.request.parser; + +import com.yahoo.search.grouping.request.AllOperation; +import com.yahoo.search.grouping.request.EachOperation; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.yql.VespaGroupingStep; +import com.yahoo.search.yql.YqlParser; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupingParserTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests. + // + // -------------------------------------------------------------------------------- + + @Test + public void requireThatMathAllowsWhitespace() { + for (String op : Arrays.asList("+", " +", " + ", "+ ", + "-", " -", " - ", "- ", + "*", " *", " * ", "* ", + "/", " /", " / ", "/ ", + "%", " %", " % ", "% ")) { + assertParse("all(group(foo " + op + " 69) each(output(count())))"); + assertParse("all(group(foo " + op + " 6 " + op + " 9) each(output(count())))"); + assertParse("all(group(69 " + op + " foo) each(output(count())))"); + assertParse("all(group(6 " + op + " 9 " + op + " foo) each(output(count())))"); + } + } + + @Test + public void testRequestList() { + List<GroupingOperation> lst = GroupingOperation.fromStringAsList("all();each();all() where(true);each()"); + assertNotNull(lst); + assertEquals(4, lst.size()); + assertTrue(lst.get(0) instanceof AllOperation); + assertTrue(lst.get(1) instanceof EachOperation); + assertTrue(lst.get(2) instanceof AllOperation); + assertTrue(lst.get(3) instanceof EachOperation); + } + + @Test + public void testAttributeFunctions() { + assertParse("all(group(foo) each(output(sum(attribute(bar)))))", + "all(group(foo) each(output(sum(attribute(bar)))))"); + assertParse("all(group(foo) each(output(sum(interpolatedlookup(bar, 0.25)))))", + "all(group(foo) each(output(sum(interpolatedlookup(bar, 0.25)))))"); + assertParse("all(group(foo) each(output(sum(array.at(bar, 42.0)))))", + "all(group(foo) each(output(sum(array.at(bar, 42.0)))))"); + } + + @Test + public void requireThatTokenImagesAreNotReservedWords() { + List<String> images = Arrays.asList("acos", + "acosh", + "accuracy", + "add", + "alias", + "all", + "and", + "array", + "as", + "at", + "asin", + "asinh", + "atan", + "atanh", + "attribute", + "avg", + "bucket", + "cat", + "cbrt", + "cos", + "cosh", + "count", + "debugwait", + "div", + "docidnsspecific", + "each", + "exp", + "fixedwidth", + "floor", + "group", + "hint", + "hypot", + "log", + "log1p", + "log10", + "math", + "max", + "md5", + "min", + "mod", + "mul", + "neg", + "normalizesubject", + "now", + "or", + "order", + "output", + "pow", + "precision", + "predefined", + "relevance", + "reverse", + "sin", + "sinh", + "size", + "sort", + "interpolatedlookup", + "sqrt", + "strcat", + "strlen", + "sub", + "sum", + "summary", + "tan", + "tanh", + "time", + "date", + "dayofmonth", + "dayofweek", + "dayofyear", + "hourofday", + "minuteofhour", + "monthofyear", + "secondofminute", + "year", + "todouble", + "tolong", + "toraw", + "tostring", + "true", + "false", + "uca", + "where", + "x", + "xor", + "xorbit", + "y", + "ymum", + "zcurve"); + for (String image : images) { + assertParse("all(group(" + image + "))", "all(group(" + image + "))"); + } + } + + @Test + public void testTokenizedWhitespace() { + String expected = "all(group(foo) each(output(max(bar))))"; + + assertParse(" all(group(foo)each(output(max(bar))))", expected); + assertIllegalArgument("all (group(foo)each(output(max(bar))))", "Encountered \" \" at line 1, column 4."); + assertParse("all( group(foo)each(output(max(bar))))", expected); + assertIllegalArgument("all(group (foo)each(output(max(bar))))", "Encountered \" \" at line 1, column 10."); + assertParse("all(group( foo)each(output(max(bar))))", expected); + assertParse("all(group(foo )each(output(max(bar))))", expected); + assertParse("all(group(foo) each(output(max(bar))))", expected); + assertIllegalArgument("all(group(foo)each (output(max(bar))))", "Encountered \" \" at line 1, column 19."); + assertParse("all(group(foo)each( output(max(bar))))", expected); + assertIllegalArgument("all(group(foo)each(output (max(bar))))", "Encountered \" \" at line 1, column 26."); + assertParse("all(group(foo)each(output( max(bar))))", expected); + assertParse("all(group(foo)each(output(max(bar))))", expected); + assertParse("all(group(foo)each(output(max( bar))))", expected); + assertParse("all(group(foo)each(output(max(bar ))))", expected); + assertParse("all(group(foo)each(output(max(bar) )))", expected); + assertParse("all(group(foo)each(output(max(bar)) ))", expected); + assertParse("all(group(foo)each(output(max(bar))) )", expected); + assertParse("all(group(foo)each(output(max(bar)))) ", expected); + } + + @Test + public void testOperationTypes() { + assertParse("all()"); + assertParse("each()"); + assertParse("all(each())"); + assertParse("each(all())"); + assertParse("all(all() all())"); + assertParse("all(all() each())"); + assertParse("all(each() all())"); + assertParse("all(each() each())"); + assertParse("each(all() all())"); + assertIllegalArgument("each(all() each())", + "Operation 'each()' can not operate on single hit."); + assertIllegalArgument("each(group(foo) all() each())", + "Operation 'each(group(foo) all() each())' can not group single hit."); + assertIllegalArgument("each(each() all())", + "Operation 'each()' can not operate on single hit."); + assertIllegalArgument("each(group(foo) each() all())", + "Operation 'each(group(foo) each() all())' can not group single hit."); + assertIllegalArgument("each(each() each())", + "Operation 'each()' can not operate on single hit."); + assertIllegalArgument("each(group(foo) each() each())", + "Operation 'each(group(foo) each() each())' can not group single hit."); + } + + @Test + public void testOperationParts() { + assertParse("all(group(foo))"); + assertParse("all(hint(foo))"); + assertParse("all(hint(foo) hint(bar))"); + assertParse("all(max(1))"); + assertParse("all(order(foo))"); + assertParse("all(order(+foo))"); + assertParse("all(order(-foo))"); + assertParse("all(order(foo, bar))"); + assertParse("all(order(foo, +bar))"); + assertParse("all(order(foo, -bar))"); + assertParse("all(order(+foo, bar))"); + assertParse("all(order(+foo, +bar))"); + assertParse("all(order(+foo, -bar))"); + assertParse("all(order(-foo, bar))"); + assertParse("all(order(-foo, +bar))"); + assertParse("all(order(-foo, -bar))"); + assertParse("all(output(min(a)))"); + assertParse("all(output(min(a), min(b)))"); + assertParse("all(precision(1))"); + assertParse("all(where(foo))"); + assertParse("all(where($foo))"); + } + + @Test + public void testComplexExpressionTypes() { + // fixedwidth + assertParse("all(group(fixedwidth(foo, 1)))"); + assertParse("all(group(fixedwidth(foo, 1.2)))"); + + // md5 + assertParse("all(group(md5(foo, 1)))"); + + // predefined + assertParse("all(group(predefined(foo, bucket(1, 2))))"); + assertParse("all(group(predefined(foo, bucket(-1, 2))))"); + assertParse("all(group(predefined(foo, bucket(-2, -1))))"); + assertParse("all(group(predefined(foo, bucket(1, 2), bucket(3, 4))))"); + assertParse("all(group(predefined(foo, bucket(1, 2), bucket(3, 4), bucket(5, 6))))"); + assertParse("all(group(predefined(foo, bucket(1, 2), bucket(2, 3), bucket(3, 4))))"); + assertParse("all(group(predefined(foo, bucket(-100, 0), bucket(0), bucket<0, 100))))"); + + assertParse("all(group(predefined(foo, bucket[1, 2>)))"); + assertParse("all(group(predefined(foo, bucket[-1, 2>)))"); + assertParse("all(group(predefined(foo, bucket[-2, -1>)))"); + assertParse("all(group(predefined(foo, bucket[1, 2>, bucket(3, 4>)))"); + assertParse("all(group(predefined(foo, bucket[1, 2>, bucket[3, 4>, bucket[5, 6>)))"); + assertParse("all(group(predefined(foo, bucket[1, 2>, bucket[2, 3>, bucket[3, 4>)))"); + + assertParse("all(group(predefined(foo, bucket<1, 5>)))"); + assertParse("all(group(predefined(foo, bucket[1, 5>)))"); + assertParse("all(group(predefined(foo, bucket<1, 5])))"); + assertParse("all(group(predefined(foo, bucket[1, 5])))"); + + assertParse("all(group(predefined(foo, bucket<1, inf>)))"); + assertParse("all(group(predefined(foo, bucket<-inf, -1>)))"); + assertParse("all(group(predefined(foo, bucket<a, inf>)))"); + assertParse("all(group(predefined(foo, bucket<'a', inf>)))"); + assertParse("all(group(predefined(foo, bucket<-inf, a>)))"); + assertParse("all(group(predefined(foo, bucket[-inf, 'a'>)))"); + assertParse("all(group(predefined(foo, bucket<-inf, -0.3>)))"); + assertParse("all(group(predefined(foo, bucket<0.3, inf])))"); + assertParse("all(group(predefined(foo, bucket<0.3, inf])))"); + assertParse("all(group(predefined(foo, bucket<infinite, inf])))"); + assertParse("all(group(predefined(foo, bucket<myinf, inf])))"); + assertParse("all(group(predefined(foo, bucket<-inf, infinite])))"); + assertParse("all(group(predefined(foo, bucket<-inf, myinf])))"); + + assertParse("all(group(predefined(foo, bucket(1.0, 2.0))))"); + assertParse("all(group(predefined(foo, bucket(1.0, 2.0), bucket(3.0, 4.0))))"); + assertParse("all(group(predefined(foo, bucket(1.0, 2.0), bucket(3.0, 4.0), bucket(5.0, 6.0))))"); + + assertParse("all(group(predefined(foo, bucket<1.0, 2.0>)))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0>)))"); + assertParse("all(group(predefined(foo, bucket<1.0, 2.0])))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0])))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0>, bucket[3.0, 4.0>)))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0>, bucket[3.0, 4.0>, bucket[5.0, 6.0>)))"); + assertParse("all(group(predefined(foo, bucket[1.0, 2.0>, bucket[2.0], bucket<2.0, 6.0>)))"); + + assertParse("all(group(predefined(foo, bucket('a', 'b'))))"); + assertParse("all(group(predefined(foo, bucket['a', 'b'>)))"); + assertParse("all(group(predefined(foo, bucket<'a', 'c'>)))"); + assertParse("all(group(predefined(foo, bucket<'a', 'b'])))"); + assertParse("all(group(predefined(foo, bucket['a', 'b'])))"); + assertParse("all(group(predefined(foo, bucket('a', 'b'), bucket('c', 'd'))))"); + assertParse("all(group(predefined(foo, bucket('a', 'b'), bucket('c', 'd'), bucket('e', 'f'))))"); + + assertParse("all(group(predefined(foo, bucket(\"a\", \"b\"))))"); + assertParse("all(group(predefined(foo, bucket(\"a\", \"b\"), bucket(\"c\", \"d\"))))"); + assertParse("all(group(predefined(foo, bucket(\"a\", \"b\"), bucket(\"c\", \"d\"), bucket(\"e\", \"f\"))))"); + + assertParse("all(group(predefined(foo, bucket(a, b))))"); + assertParse("all(group(predefined(foo, bucket(a, b), bucket(c, d))))"); + assertParse("all(group(predefined(foo, bucket(a, b), bucket(c, d), bucket(e, f))))"); + assertParse("all(group(predefined(foo, bucket(a, b), bucket(c), bucket(e, f))))"); + + assertParse("all(group(predefined(foo, bucket('a', \"b\"))))"); + assertParse("all(group(predefined(foo, bucket('a', \"b\"), bucket(c, 'd'))))"); + assertParse("all(group(predefined(foo, bucket('a', \"b\"), bucket(c, 'd'), bucket(\"e\", f))))"); + + assertParse("all(group(predefined(foo, bucket('a(', \"b)\"), bucket(c, 'd()'))))"); + assertParse("all(group(predefined(foo, bucket({2}, {6}), bucket({7}, {12}))))"); + assertParse("all(group(predefined(foo, bucket({0, 2}, {0, 6}), bucket({0, 7}, {0, 12}))))"); + assertParse("all(group(predefined(foo, bucket({'b', 'a'}, {'k', 'a'}), bucket({'k', 'a'}, {'u', 'b'}))))"); + + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2.0))))", + "Bucket type mismatch, expected 'LongValue' got 'DoubleValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, '2'))))", + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2), bucket(3.0, 4.0))))", + "Bucket type mismatch, expected 'LongValue' got 'DoubleValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2), bucket('3', '4'))))", + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2), bucket(\"3\", \"4\"))))", + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket(1, 2), bucket(three, four))))", + "Bucket type mismatch, expected 'LongValue' got 'StringValue'."); + assertIllegalArgument("all(group(predefined(foo, bucket<-inf, inf>)))", + "Bucket type mismatch, cannot both be infinity"); + assertIllegalArgument("all(group(predefined(foo, bucket<inf, -inf>)))", + "Encountered \"inf\" at line 1, column 34."); + + assertIllegalArgument("all(group(predefined(foo, bucket(2, 1))))", + "Bucket to-value can not be less than from-value."); + assertIllegalArgument("all(group(predefined(foo, bucket(3, 4), bucket(1, 2))))", + "Buckets must be monotonically increasing, got bucket[3, 4> before bucket[1, 2>."); + assertIllegalArgument("all(group(predefined(foo, bucket(b, a))))", + "Bucket to-value can not be less than from-value."); + assertIllegalArgument("all(group(predefined(foo, bucket(b, -inf))))", + "Encountered \"-inf\" at line 1, column 37."); + assertIllegalArgument("all(group(predefined(foo, bucket(c, d), bucket(a, b))))", + "Buckets must be monotonically increasing, got bucket[\"c\", \"d\"> before bucket[\"a\", \"b\">."); + assertIllegalArgument("all(group(predefined(foo, bucket(c, d), bucket(-inf, e))))", + "Buckets must be monotonically increasing, got bucket[\"c\", \"d\"> before bucket[-inf, \"e\">."); + assertIllegalArgument("all(group(predefined(foo, bucket(u, inf), bucket(e, i))))", + "Buckets must be monotonically increasing, got bucket[\"u\", inf> before bucket[\"e\", \"i\">."); + + // xorbit + assertParse("all(group(xorbit(foo, 1)))"); + } + + @Test + public void testInfixArithmetic() { + assertParse("all(group(1))", "all(group(1))"); + assertParse("all(group(1+2))", "all(group(add(1, 2)))"); + assertParse("all(group(1-2))", "all(group(sub(1, 2)))"); + assertParse("all(group(1*2))", "all(group(mul(1, 2)))"); + assertParse("all(group(1/2))", "all(group(div(1, 2)))"); + assertParse("all(group(1%2))", "all(group(mod(1, 2)))"); + assertParse("all(group(1+2+3))", "all(group(add(add(1, 2), 3)))"); + assertParse("all(group(1+2-3))", "all(group(add(1, sub(2, 3))))"); + assertParse("all(group(1+2*3))", "all(group(add(1, mul(2, 3))))"); + assertParse("all(group(1+2/3))", "all(group(add(1, div(2, 3))))"); + assertParse("all(group(1+2%3))", "all(group(add(1, mod(2, 3))))"); + assertParse("all(group((1+2)+3))", "all(group(add(add(1, 2), 3)))"); + assertParse("all(group((1+2)-3))", "all(group(sub(add(1, 2), 3)))"); + assertParse("all(group((1+2)*3))", "all(group(mul(add(1, 2), 3)))"); + assertParse("all(group((1+2)/3))", "all(group(div(add(1, 2), 3)))"); + assertParse("all(group((1+2)%3))", "all(group(mod(add(1, 2), 3)))"); + } + + @Test + public void testOperationLabel() { + assertParse("each() as(foo)", + "each() as(foo)"); + assertParse("all(each() as(foo)" + + " each() as(bar))", + "all(each() as(foo) each() as(bar))"); + assertParse("all(group(a) each(each() as(foo)" + + " each() as(bar))" + + " each() as(baz))", + "all(group(a) each(each() as(foo) each() as(bar)) each() as(baz))"); + + assertIllegalArgument("all() as(foo)", "Encountered \"as\" at line 1, column 7."); + assertIllegalArgument("all(all() as(foo))", "Encountered \"as\" at line 1, column 11."); + assertIllegalArgument("each(all() as(foo))", "Encountered \"as\" at line 1, column 12."); + } + + @Test + public void testAttributeName() { + assertParse("all(group(foo))"); + assertIllegalArgument("all(group(foo.))", + "Encountered \")\" at line 1, column 15."); + assertParse("all(group(foo.bar))"); + assertIllegalArgument("all(group(foo.bar.))", + "Encountered \")\" at line 1, column 19."); + assertParse("all(group(foo.bar.baz))"); + } + + @Test + public void testOutputLabel() { + assertParse("all(output(min(a) as(foo)))", + "all(output(min(a) as(foo)))"); + assertParse("all(output(min(a) as(foo), max(b) as(bar)))", + "all(output(min(a) as(foo), max(b) as(bar)))"); + + assertIllegalArgument("all(output(min(a)) as(foo))", + "Encountered \"as\" at line 1, column 20."); + } + + @Test + public void testRootWhere() { + String expected = "all(where(bar) all(group(foo)))"; + assertParse("all(where(bar) all(group(foo)))", expected); + assertParse("all(group(foo)) where(bar)", expected); + } + + @Test + public void testParseBadRequest() { + assertIllegalArgument("output(count())", + "Encountered \"output\" at line 1, column 1."); + assertIllegalArgument("each(output(count()))", + "Expression 'count()' not applicable for single hit."); + assertIllegalArgument("all(output(count())))", + "Encountered \")\" at line 1, column 21."); + } + + @Test + public void testAttributeFunction() { + assertParse("all(group(attribute(foo)))"); + assertParse("all(group(attribute(foo)) order(sum(attribute(a))))"); + } + + @Test + public void testAccuracy() { + assertParse("all(accuracy(0.5))"); + assertParse("all(group(foo) accuracy(1.0))"); + } + + @Test + public void testMapSyntax() { + assertParse("all(group(my.little{key}))", "all(group(my.little{\"key\"}))"); + assertParse("all(group(my.little{key }))", "all(group(my.little{\"key\"}))"); + assertParse("all(group(my.little{\"key\"}))", "all(group(my.little{\"key\"}))"); + assertParse("all(group(my.little{\"key{}%\"}))", "all(group(my.little{\"key{}%\"}))"); + } + + @Test + public void testMisc() { + for (String fnc : Arrays.asList("time.date", + "time.dayofmonth", + "time.dayofweek", + "time.dayofyear", + "time.hourofday", + "time.minuteofhour", + "time.monthofyear", + "time.secondofminute", + "time.year")) { + assertParse("each(output(" + fnc + "(foo)))"); + } + + assertParse("all(group(artist) max(7))"); + assertParse("all(max(76) all(group(artist) max(7)))"); + assertParse("all(group(artist) max(7) where(true))"); + assertParse("all(group(artist) order(sum(a)) output(count()))"); + assertParse("all(group(artist) order(+sum(a)) output(count()))"); + assertParse("all(group(artist) order(-sum(a)) output(count()))"); + assertParse("all(group(artist) order(-sum(a), +xor(b)) output(count()))"); + assertParse("all(group(artist) max(1) output(count()))"); + assertParse("all(group(-(m)) max(1) output(count()))"); + assertParse("all(group(min) max(1) output(count()))"); + assertParse("all(group(artist) max(2) each(each(output(summary()))))"); + assertParse("all(group(artist) max(2) each(each(output(summary(simple)))))"); + assertParse("all(group(artist) max(5) each(output(count()) each(output(summary()))))"); + assertParse("all(group(ymum()))"); + assertParse("all(group(strlen(attr)))"); + assertParse("all(group(normalizesubject(attr)))"); + assertParse("all(group(strcat(attr, attr2)))"); + assertParse("all(group(tostring(attr)))"); + assertParse("all(group(toraw(attr)))"); + assertParse("all(group(zcurve.x(attr)))"); + assertParse("all(group(zcurve.y(attr)))"); + assertParse("all(group(uca(attr, \"foo\")))"); + assertParse("all(group(uca(attr, \"foo\", \"PRIMARY\")))"); + assertParse("all(group(uca(attr, \"foo\", \"SECONDARY\")))"); + assertParse("all(group(uca(attr, \"foo\", \"TERTIARY\")))"); + assertParse("all(group(uca(attr, \"foo\", \"QUATERNARY\")))"); + assertParse("all(group(uca(attr, \"foo\", \"IDENTICAL\")))"); + assertIllegalArgument("all(group(uca(attr, \"foo\", \"foo\")))", "Not a valid UCA strength: foo"); + assertParse("all(group(tolong(attr)))"); + assertParse("all(group(sort(attr)))"); + assertParse("all(group(reverse(attr)))"); + assertParse("all(group(docidnsspecific()))"); + assertParse("all(group(relevance()))"); + // TODO: assertParseRequest("all(group(a) each(output(xor(md5(b)) xor(md5(b, 0, 64)))))"); + // TODO: assertParseRequest("all(group(a) each(output(xor(xorbit(b)) xor(xorbit(b, 64)))))"); + assertParse("all(group(artist) each(each(output(summary()))))"); + assertParse("all(group(artist) max(13) each(group(fixedwidth(year, 21.34)) max(55) output(count()) " + + "each(each(output(summary())))))"); + assertParse("all(group(artist) max(13) each(group(predefined(year, bucket(7, 19), bucket(90, 300))) " + + "max(55) output(count()) each(each(output(summary())))))"); + assertParse("all(group(artist) max(13) each(group(predefined(year, bucket(7.1, 19.0), bucket(90.7, 300.0))) " + + "max(55) output(count()) each(each(output(summary())))))"); + assertParse("all(group(artist) max(13) each(group(predefined(year, bucket('a', 'b'), bucket('cd', 'cde'))) " + + "max(55) output(count()) each(each(output(summary())))))"); + + assertParse("all(output(count()))"); + assertParse("all(group(album) output(count()))"); + assertParse("all(group(album) each(output(count())))"); + assertParse("all(group(artist) each(group(album) output(count()))" + + " each(group(song) output(count())))"); + assertParse("all(group(artist) output(count())" + + " each(group(album) output(count())" + + " each(group(song) output(count())" + + " each(each(output(summary()))))))"); + assertParse("all(group(album) order(-$total=sum(length)) each(output($total)))"); + assertParse("all(group(album) max(1) each(output(sum(length))))"); + assertParse("all(group(artist) each(max(2) each(output(summary()))))"); + assertParse("all(group(artist) max(3)" + + " each(group(album as(albumsongs)) each(each(output(summary()))))" + + " each(group(album as(albumlength)) output(sum(sum(length)))))"); + assertParse("all(group(artist) max(15)" + + " each(group(album) " + + " each(group(song)" + + " each(max(2) each(output(summary()))))))"); + assertParse("all(group(artist) max(15)" + + " each(group(album)" + + " each(group(song)" + + " each(max(2) each(output(summary())))))" + + " each(group(song) max(5) order(sum(popularity))" + + " each(output(sum(sold)) each(output(summary())))))"); + + assertParse("all(group(artist) order(max(relevance) * count()) each(output(count())))"); + assertParse("all(group(artist) each(output(sum(popularity) / count())))"); + assertParse("all(group(artist) accuracy(0.1) each(output(sum(popularity) / count())))"); + assertParse("all(group(debugwait(artist, 3.3, true)))"); + assertParse("all(group(debugwait(artist, 3.3, false)))"); + assertIllegalArgument("all(group(debugwait(artist, -3.3, true)))", + "Encountered \"-\" at line 1, column 29"); + assertIllegalArgument("all(group(debugwait(artist, 3.3, lol)))", + "Encountered \"lol\" at line 1, column 34"); + } + + @Test + public void requireThatParseExceptionMessagesContainErrorMarker() { + assertIllegalArgument("foo", + "Encountered \"foo\" at line 1, column 1.\n" + + "Was expecting one of:\n" + + " <SPACE> ...\n" + + " \"all\" ...\n" + + " \"each\" ...\n" + + " \n" + + "At position:\n" + + "foo\n" + + "^"); + assertIllegalArgument("\n foo", + "Encountered \"foo\" at line 2, column 2.\n" + + "Was expecting one of:\n" + + " <SPACE> ...\n" + + " \"all\" ...\n" + + " \"each\" ...\n" + + " \n" + + "At position:\n" + + " foo\n" + + " ^"); + } + + // -------------------------------------------------------------------------------- + // + // Utilities. + // + // -------------------------------------------------------------------------------- + + private static void assertParse(String request, String... expectedOperations) { + List<GroupingOperation> operations = GroupingOperation.fromStringAsList(request); + List<String> actual = new ArrayList<>(operations.size()); + for (GroupingOperation operation : operations) { + operation.resolveLevel(1); + actual.add(operation.toString()); + } + if (expectedOperations.length > 0) { + assertEquals(Arrays.asList(expectedOperations), actual); + } + + // make sure that operation does not mutate through toString() -> fromString() + for (GroupingOperation operation : operations) { + assertEquals(operation.toString(), GroupingOperation.fromString(operation.toString()).toString()); + } + + // make sure that yql+ is capable of handling request + assertYqlParsable(request, expectedOperations); + } + + private static void assertYqlParsable(String request, String... expectedOperations) { + YqlParser parser = new YqlParser(new ParserEnvironment()); + parser.parse(new Parsable().setQuery("select foo from bar where baz contains 'baz' | " + request + ";")); + List<VespaGroupingStep> steps = parser.getGroupingSteps(); + List<String> actual = new ArrayList<>(steps.size()); + for (VespaGroupingStep step : steps) { + actual.add(step.getOperation().toString()); + } + if (expectedOperations.length > 0) { + assertEquals(Arrays.asList(expectedOperations), actual); + } + } + + private static void assertIllegalArgument(String request, String expectedException) { + try { + GroupingOperation.fromString(request).resolveLevel(1); + fail("Expected: " + expectedException); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage(), e.getMessage().startsWith(expectedException)); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java new file mode 100644 index 00000000000..e3bccde2767 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupIdTestCase { + + @Test + public void requireThatAccessorsWork() { + ValueGroupId valueId = new DoubleId(6.9); + assertEquals(6.9, valueId.getValue()); + BucketGroupId rangeId = new DoubleBucketId(6.0, 9.0); + assertEquals(6.0, rangeId.getFrom()); + assertEquals(9.0, rangeId.getTo()); + + valueId = new LongId(69L); + assertEquals(69L, valueId.getValue()); + rangeId = new LongBucketId(6L, 9L); + assertEquals(6L, rangeId.getFrom()); + assertEquals(9L, rangeId.getTo()); + + valueId = new RawId(new byte[] { 6, 9 }); + assertArrayEquals(new byte[] { 6, 9 }, (byte[])valueId.getValue()); + rangeId = new RawBucketId(new byte[] { 6, 9 }, new byte[] { 9, 6 }); + assertArrayEquals(new byte[] { 6, 9 }, (byte[])rangeId.getFrom()); + assertArrayEquals(new byte[] { 9, 6 }, (byte[])rangeId.getTo()); + + valueId = new StringId("69"); + assertEquals("69", valueId.getValue()); + rangeId = new StringBucketId("6", "9"); + assertEquals("6", rangeId.getFrom()); + assertEquals("9", rangeId.getTo()); + } + + @Test + public void requireThatToStringCorrespondsToType() { + assertEquals("group:double:6.9", new DoubleId(6.9).toString()); + assertEquals("group:double_bucket:6.0:9.0", new DoubleBucketId(6.0, 9.0).toString()); + assertEquals("group:long:69", new LongId(69L).toString()); + assertEquals("group:long_bucket:6:9", new LongBucketId(6L, 9L).toString()); + assertEquals("group:null", new NullId().toString()); + assertEquals("group:raw:[6, 9]", new RawId(new byte[] { 6, 9 }).toString()); + assertEquals("group:raw_bucket:[6, 9]:[9, 6]", new RawBucketId(new byte[] { 6, 9 }, new byte[] { 9, 6 }).toString()); + assertTrue(new RootId(0).toString().startsWith("group:root:")); + assertEquals("group:string:69", new StringId("69").toString()); + assertEquals("group:string_bucket:6:9", new StringBucketId("6", "9").toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java new file mode 100644 index 00000000000..c9aa0848a8b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupListTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupListTestCase { + + @Test + public void requireThatAccessorsWork() { + GroupList lst = new GroupList("foo"); + assertEquals("foo", lst.getLabel()); + assertEquals(0, lst.continuations().size()); + + MyContinuation foo = new MyContinuation(); + lst.continuations().put("foo", foo); + assertEquals(1, lst.continuations().size()); + assertSame(foo, lst.continuations().get("foo")); + + MyContinuation bar = new MyContinuation(); + lst.continuations().put("bar", bar); + assertEquals(2, lst.continuations().size()); + assertSame(bar, lst.continuations().get("bar")); + } + + private static class MyContinuation extends Continuation { + + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupTestCase.java new file mode 100644 index 00000000000..9acab986ac2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupTestCase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupTestCase { + + @Test + public void requireThatListsAreAccessibleByLabel() { + Group grp = new Group(new LongId(69L), new Relevance(1)); + grp.add(new Hit("hit")); + grp.add(new HitList("hitList")); + grp.add(new GroupList("groupList")); + + assertNotNull(grp.getGroupList("groupList")); + assertNull(grp.getGroupList("unknownGroupList")); + assertNull(grp.getGroupList("hitList")); + + assertNotNull(grp.getHitList("hitList")); + assertNull(grp.getHitList("unknownHitList")); + assertNull(grp.getHitList("groupList")); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java new file mode 100644 index 00000000000..f9f7047abc0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/HitListTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class HitListTestCase { + + @Test + public void requireThatAccessorsWork() { + HitList lst = new HitList("foo"); + assertEquals("foo", lst.getLabel()); + assertEquals(0, lst.continuations().size()); + + MyContinuation foo = new MyContinuation(); + lst.continuations().put("foo", foo); + assertEquals(1, lst.continuations().size()); + assertSame(foo, lst.continuations().get("foo")); + + MyContinuation bar = new MyContinuation(); + lst.continuations().put("bar", bar); + assertEquals(2, lst.continuations().size()); + assertSame(bar, lst.continuations().get("bar")); + } + + private static class MyContinuation extends Continuation { + + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java new file mode 100644 index 00000000000..97a2e81d9ba --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java @@ -0,0 +1,174 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.result; + +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.result.Relevance; +import com.yahoo.text.Utf8; +import com.yahoo.text.XMLWriter; +import org.junit.Test; + +import java.io.IOException; +import java.io.StringWriter; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class HitRendererTestCase { + + @Test + public void requireThatGroupListsRenderAsExpected() { + assertRender(new GroupList("foo"), "<grouplist label=\"foo\"></grouplist>\n"); + assertRender(new GroupList("b\u00e6z"), "<grouplist label=\"b\u00e6z\"></grouplist>\n"); + + GroupList lst = new GroupList("foo"); + lst.continuations().put("bar.key", new MyContinuation("bar.val")); + lst.continuations().put("baz.key", new MyContinuation("baz.val")); + assertRender(lst, "<grouplist label=\"foo\">\n" + + "<continuation id=\"bar.key\">bar.val</continuation>\n" + + "<continuation id=\"baz.key\">baz.val</continuation>\n" + + "</grouplist>\n"); + } + + @Test + public void requireThatGroupIdsRenderAsExpected() { + assertRender(newGroup(new DoubleId(6.9)), + "<group relevance=\"1.0\">\n" + + "<id type=\"double\">6.9</id>\n" + + "</group>\n"); + assertRender(newGroup(new LongId(69L)), + "<group relevance=\"1.0\">\n" + + "<id type=\"long\">69</id>\n" + + "</group>\n"); + assertRender(newGroup(new NullId()), + "<group relevance=\"1.0\">\n" + + "<id type=\"null\"/>\n" + + "</group>\n"); + assertRender(newGroup(new RawId(Utf8.toBytes("foo"))), + "<group relevance=\"1.0\">\n" + + "<id type=\"raw\">[102, 111, 111]</id>\n" + + "</group>\n"); + assertRender(newGroup(new StringId("foo")), + "<group relevance=\"1.0\">\n" + + "<id type=\"string\">foo</id>\n" + + "</group>\n"); + assertRender(newGroup(new StringId("b\u00e6z")), + "<group relevance=\"1.0\">\n" + + "<id type=\"string\">b\u00e6z</id>\n" + + "</group>\n"); + assertRender(newGroup(new DoubleBucketId(6.9, 9.6)), + "<group relevance=\"1.0\">\n" + + "<id type=\"double_bucket\">\n<from>6.9</from>\n<to>9.6</to>\n</id>\n" + + "</group>\n"); + assertRender(newGroup(new LongBucketId(6L, 9L)), + "<group relevance=\"1.0\">\n" + + "<id type=\"long_bucket\">\n<from>6</from>\n<to>9</to>\n</id>\n" + + "</group>\n"); + assertRender(newGroup(new StringBucketId("bar", "baz")), + "<group relevance=\"1.0\">\n" + + "<id type=\"string_bucket\">\n<from>bar</from>\n<to>baz</to>\n</id>\n" + + "</group>\n"); + assertRender(newGroup(new StringBucketId("b\u00e6r", "b\u00e6z")), + "<group relevance=\"1.0\">\n" + + "<id type=\"string_bucket\">\n<from>b\u00e6r</from>\n<to>b\u00e6z</to>\n</id>\n" + + "</group>\n"); + assertRender(newGroup(new RawBucketId(Utf8.toBytes("bar"), Utf8.toBytes("baz"))), + "<group relevance=\"1.0\">\n" + + "<id type=\"raw_bucket\">\n<from>[98, 97, 114]</from>\n<to>[98, 97, 122]</to>\n</id>\n" + + "</group>\n"); + } + + @Test + public void requireThatGroupsRenderAsExpected() { + Group group = newGroup(new StringId("foo")); + group.setField("foo", "bar"); + group.setField("baz", "cox"); + assertRender(group, "<group relevance=\"1.0\">\n" + + "<id type=\"string\">foo</id>\n" + + "<output label=\"foo\">bar</output>\n" + + "<output label=\"baz\">cox</output>\n" + + "</group>\n"); + + group = newGroup(new StringId("foo")); + group.setField("foo", "b\u00e6r"); + group.setField("b\u00e5z", "cox"); + assertRender(group, "<group relevance=\"1.0\">\n" + + "<id type=\"string\">foo</id>\n" + + "<output label=\"foo\">b\u00e6r</output>\n" + + "<output label=\"b\u00e5z\">cox</output>\n" + + "</group>\n"); + } + + @Test + public void requireThatRootGroupsRenderAsExpected() { + RootGroup group = new RootGroup(0, new MyContinuation("69")); + group.setField("foo", "bar"); + group.setField("baz", "cox"); + assertRender(group, "<group relevance=\"1.0\">\n" + + "<id type=\"root\"/>\n" + + "<continuation id=\"this\">69</continuation>\n" + + "<output label=\"foo\">bar</output>\n" + + "<output label=\"baz\">cox</output>\n" + + "</group>\n"); + + group = new RootGroup(0, new MyContinuation("96")); + group.setField("foo", "b\u00e6r"); + group.setField("b\u00e5z", "cox"); + assertRender(group, "<group relevance=\"1.0\">\n" + + "<id type=\"root\"/>\n" + + "<continuation id=\"this\">96</continuation>\n" + + "<output label=\"foo\">b\u00e6r</output>\n" + + "<output label=\"b\u00e5z\">cox</output>\n" + + "</group>\n"); + } + + @Test + public void requireThatHitListsRenderAsExpected() { + assertRender(new HitList("foo"), "<hitlist label=\"foo\"></hitlist>\n"); + assertRender(new HitList("b\u00e6z"), "<hitlist label=\"b\u00e6z\"></hitlist>\n"); + + HitList lst = new HitList("foo"); + lst.continuations().put("bar.key", new MyContinuation("bar.val")); + lst.continuations().put("baz.key", new MyContinuation("baz.val")); + assertRender(lst, "<hitlist label=\"foo\">\n" + + "<continuation id=\"bar.key\">bar.val</continuation>\n" + + "<continuation id=\"baz.key\">baz.val</continuation>\n" + + "</hitlist>\n"); +} + + private static Group newGroup(GroupId id) { + return new Group(id, new Relevance(1)); + } + + @SuppressWarnings("deprecation") + private static void assertRender(HitGroup hit, String expectedXml) { + StringWriter str = new StringWriter(); + XMLWriter out = new XMLWriter(str, 0, -1); + try { + HitRenderer.renderHeader(hit, out); + while (out.openTags().size() > 0) { + out.closeTag(); + } + } catch (IOException e) { + fail(); + } + assertEquals(expectedXml, str.toString()); + } + + private static class MyContinuation extends Continuation { + + final String str; + + MyContinuation(String str) { + this.str = str; + } + + @Override + public String toString() { + return str; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java new file mode 100644 index 00000000000..5d2d584af9b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/CompositeContinuationTestCase.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class CompositeContinuationTestCase { + + @Test + public void requireThatAccessorsWork() { + CompositeContinuation cnt = new CompositeContinuation(); + Iterator<EncodableContinuation> it = cnt.iterator(); + assertFalse(it.hasNext()); + + EncodableContinuation foo = new MyContinuation(); + cnt.add(foo); + it = cnt.iterator(); + assertTrue(it.hasNext()); + assertSame(foo, it.next()); + assertFalse(it.hasNext()); + + EncodableContinuation bar = new MyContinuation(); + cnt.add(bar); + it = cnt.iterator(); + assertTrue(it.hasNext()); + assertSame(foo, it.next()); + assertTrue(it.hasNext()); + assertSame(bar, it.next()); + assertFalse(it.hasNext()); + } + + @Test + public void requireThatCompositeContinuationsAreFlattened() { + assertEncode("BCBCBCBEBGBCBKCBACBKCCK", + newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21))); + assertEncode("BCBBBBBDBFBCBJBPCBJCCJ", + newComposite(newComposite(newOffset(-1, -1, -2, -3)), newComposite(newOffset(-5, -8, -13, -21)))); + } + + @Test + public void requireThatEmptyStringCanBeDecoded() { + assertDecode("", new CompositeContinuation()); + } + + @Test + public void requireThatCompositeContinuationsCanBeDecoded() { + assertDecode("BCBCBCBEBGBCBKCBACBKCCK", + newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21))); + assertDecode("BCBBBBBDBFBCBJBPCBJCCJ", + newComposite(newOffset(-1, -1, -2, -3), newOffset(-5, -8, -13, -21))); + } + + @Test + public void requireThatHashCodeIsImplemented() { + assertEquals(newComposite().hashCode(), newComposite().hashCode()); + } + + @Test + public void requireThatEqualsIsImplemented() { + CompositeContinuation cnt = newComposite(); + assertFalse(cnt.equals(new Object())); + assertEquals(cnt, newComposite()); + assertFalse(cnt.equals(newComposite(newOffset(1, 1, 2, 3)))); + assertFalse(cnt.equals(newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21)))); + assertFalse(cnt.equals(newComposite(newOffset(5, 8, 13, 21)))); + + cnt = newComposite(newOffset(1, 1, 2, 3)); + assertFalse(cnt.equals(new Object())); + assertEquals(cnt, newComposite(newOffset(1, 1, 2, 3))); + assertFalse(cnt.equals(newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21)))); + assertFalse(cnt.equals(newComposite(newOffset(5, 8, 13, 21)))); + + cnt = newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21)); + assertFalse(cnt.equals(new Object())); + assertFalse(cnt.equals(newComposite(newOffset(1, 1, 2, 3)))); + assertEquals(cnt, newComposite(newOffset(1, 1, 2, 3), newOffset(5, 8, 13, 21))); + assertFalse(cnt.equals(newComposite(newOffset(5, 8, 13, 21)))); + } + + private static CompositeContinuation newComposite(EncodableContinuation... children) { + CompositeContinuation ret = new CompositeContinuation(); + for (EncodableContinuation child : children) { + ret.add(child); + } + return ret; + } + + private static OffsetContinuation newOffset(int resultId, int tag, int offset, int flags) { + return new OffsetContinuation(ResultId.valueOf(resultId), tag, offset, flags); + } + + private static void assertEncode(String expected, EncodableContinuation toEncode) { + IntegerEncoder actual = new IntegerEncoder(); + toEncode.encode(actual); + assertEquals(expected, actual.toString()); + } + + private static void assertDecode(String toDecode, Continuation expected) { + assertEquals(expected, ContinuationDecoder.decode(toDecode)); + } + + private static class MyContinuation extends EncodableContinuation { + + @Override + public void encode(IntegerEncoder out) { + + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java new file mode 100644 index 00000000000..386e8346cae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java @@ -0,0 +1,765 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.container.protect.Error; +import com.yahoo.document.DocumentId; +import com.yahoo.document.GlobalId; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.GroupingListHit; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.request.AllOperation; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.grouping.result.Group; +import com.yahoo.search.grouping.result.GroupList; +import com.yahoo.search.grouping.result.HitList; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.searchlib.aggregation.CountAggregationResult; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.searchlib.aggregation.HitsAggregationResult; +import com.yahoo.searchlib.aggregation.MaxAggregationResult; +import com.yahoo.searchlib.aggregation.MinAggregationResult; +import com.yahoo.searchlib.expression.AggregationRefNode; +import com.yahoo.searchlib.expression.ConstantNode; +import com.yahoo.searchlib.expression.IntegerResultNode; +import com.yahoo.searchlib.expression.StringResultNode; + +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupingExecutorTestCase { + + // -------------------------------------------------------------------------------- + // + // Tests + // + // -------------------------------------------------------------------------------- + + @Test + public void requireThatNullRequestsPass() { + Result res = newExecution(new GroupingExecutor()).search(newQuery()); + assertNotNull(res); + assertEquals(0, res.hits().size()); + } + + @Test + public void requireThatEmptyRequestsPass() { + Query query = newQuery(); + GroupingRequest.newInstance(query).setRootOperation(new AllOperation()); + Result res = newExecution(new GroupingExecutor()).search(query); + assertNotNull(res); + assertEquals(0, res.hits().size()); + } + + @Test + public void requireThatRequestsAreTransformed() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + try { + newExecution(new GroupingExecutor(), new GroupingListThrower()).search(query); + fail(); + } catch (GroupingListException e) { + assertNotNull(e.lst); + assertEquals(1, e.lst.size()); + Grouping grp = e.lst.get(0); + assertNotNull(grp); + } + } + + @Test + public void requireThatEachBelowAllDoesNotBlowUp() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(each(output(summary(bar))))")); + Result res = newExecution(new GroupingExecutor()).search(query); + assertNotNull(res); + assertEquals(1, res.hits().size()); + } + + @Test + public void requireThatSearchIsMultiPass() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + PassCounter cnt = new PassCounter(); + newExecution(new GroupingExecutor(), cnt).search(query); + assertEquals(2, cnt.numPasses); + } + + @Test + public void requireThatPassRequestsSingleLevel() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + GroupingCollector clt = new GroupingCollector(); + newExecution(new GroupingExecutor(), clt).search(query); + assertEquals(2, clt.lst.size()); + Grouping grp = clt.lst.get(0); + assertEquals(0, grp.getFirstLevel()); + assertEquals(0, grp.getLastLevel()); + grp = clt.lst.get(1); + assertEquals(1, grp.getFirstLevel()); + assertEquals(1, grp.getLastLevel()); + } + + @Test + public void requireThatAggregationPerHitWithoutGroupingDoesNotWorkYet() { + try { + execute("each(output(strlen(customer)))"); + fail(); + } catch (UnsupportedOperationException e) { + + } + } + + @Test + public void requireThatAggregationWithoutGroupingWorks() { + List<Grouping> groupings=execute("all(output(count()))"); + assertEquals(1,groupings.size()); + assertEquals(0, groupings.get(0).getLevels().size()); + assertEquals(ConstantNode.class, groupings.get(0).getRoot().getAggregationResults().get(0).getExpression().getClass()); + } + + @Test + public void requireThatGroupingIsParallel() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))) as(max)" + + " each(output(min(bar))) as(min))")); + GroupingCounter cnt = new GroupingCounter(); + newExecution(new GroupingExecutor(), cnt).search(query); + assertEquals(2, cnt.passList.size()); + assertEquals(2, cnt.passList.get(0).intValue()); + assertEquals(2, cnt.passList.get(1).intValue()); + } + + @Test + public void requireThatParallelGroupingIsNotRedundant() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))) as(shallow)" + + " each(group(baz) each(output(max(cox)))) as(deep))")); + GroupingCounter cnt = new GroupingCounter(); + newExecution(new GroupingExecutor(), cnt).search(query); + assertEquals(3, cnt.passList.size()); + assertEquals(2, cnt.passList.get(0).intValue()); + assertEquals(2, cnt.passList.get(1).intValue()); + assertEquals(1, cnt.passList.get(2).intValue()); + } + + @Test + public void requireThatPassResultsAreMerged() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(min(bar), max(bar))))")); + + Grouping grpA = new Grouping(0); + grpA.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("uniqueA")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(6)).setTag(4))) + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("common")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(9)).setTag(4))) + ); + Grouping grpB = new Grouping(0); + grpB.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("uniqueB")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(9)).setTag(4))) + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("common")).addAggregationResult(new MinAggregationResult().setMin(new IntegerResultNode(6)).setTag(3))) + ); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grpA), null), + new GroupingListHit(Arrays.asList(grpB), null)))); + Group grp = req.getResultGroup(exec.search(query)); + assertEquals(1, grp.size()); + Hit hit = grp.get(0); + assertTrue(hit instanceof GroupList); + GroupList lst = (GroupList)hit; + assertEquals(3, lst.size()); + assertNotNull(hit = lst.get("group:string:uniqueA")); + assertEquals(6L, hit.getField("max(bar)")); + assertNotNull(hit = lst.get("group:string:uniqueB")); + assertEquals(9L, hit.getField("max(bar)")); + assertNotNull(hit = lst.get("group:string:common")); + assertEquals(6L, hit.getField("min(bar)")); + assertEquals(9L, hit.getField("max(bar)")); + } + + @Test + public void requireThatUnexpectedGroupingResultsAreIgnored() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(max(bar))))")); + + Grouping grpExpected = new Grouping(0); + grpExpected.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("expected")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(69)).setTag(3))) + ); + Grouping grpUnexpected = new Grouping(1); + grpUnexpected.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("unexpected")).addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(96)).setTag(3))) + ); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grpExpected), null), + new GroupingListHit(Arrays.asList(grpUnexpected), null)))); + Group grp = req.getResultGroup(exec.search(query)); + assertEquals(1, grp.size()); + Hit hit = grp.get(0); + assertTrue(hit instanceof GroupList); + GroupList lst = (GroupList)hit; + assertEquals(1, lst.size()); + assertNotNull(hit = lst.get("group:string:expected")); + assertEquals(69L, hit.getField("max(bar)")); + assertNull(lst.get("group:string:unexpected")); + } + + @Test + public void requireThatHitsAreFilled() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar)))))")); + + Grouping grp0 = new Grouping(0); + grp0.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")) + )); + Grouping grp1 = new Grouping(0); + grp1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar").addHit(new com.yahoo.searchlib.aggregation.FS4Hit())) + )); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp0), null), + new GroupingListHit(Arrays.asList(grp1), null))), + new FillRequestThrower()); + Result res = exec.search(query); + try { + exec.fill(res); + fail(); + } catch (FillRequestException e) { + assertEquals("bar", e.summaryClass); + } + } + + @Test + public void requireThatUnfilledHitsRenderError() throws IOException { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar)))))")); + + Grouping grp0 = new Grouping(0); + grp0.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")))); + Grouping grp1 = new Grouping(0); + grp1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult( + new HitsAggregationResult(1, "bar") + .addHit(new com.yahoo.searchlib.aggregation.FS4Hit())))); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp0), null), + new GroupingListHit(Arrays.asList(grp1), null))), + new FillErrorProvider()); + Result res = exec.search(query); + exec.fill(res); + assertNotNull(res.hits().getError()); + } + + @Test + public void requireThatGroupRelevanceCanBeSynthesized() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) order(count()) each(output(count())))")); + + Grouping grp = new Grouping(0); + grp.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("foo")) + .addAggregationResult(new CountAggregationResult(1)) + .addOrderBy(new AggregationRefNode(0), true)) + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("bar")) + .addAggregationResult(new CountAggregationResult(2)) + .addOrderBy(new AggregationRefNode(0), true))); + Result res = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp), null), + new GroupingListHit(Arrays.asList(grp), null)))).search(query); + + GroupList groupList = (GroupList)req.getResultGroup(res).get(0); + assertEquals(1.0, groupList.get(0).getRelevance().getScore(), 1E-6); + assertEquals(0.5, groupList.get(1).getRelevance().getScore(), 1E-6); + } + + @Test + public void requireThatErrorsAreHandled() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar)))))")); + + Grouping grp0 = new Grouping(0); + grp0.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")) + )); + Grouping grp1 = new Grouping(0); + grp1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar").addHit(new com.yahoo.searchlib.aggregation.FS4Hit())) + )); + + ErrorProvider err = new ErrorProvider(1); + Execution exec = newExecution(new GroupingExecutor(), + err, + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp0), null), + new GroupingListHit(Arrays.asList(grp1), null)))); + Result res = exec.search(query); + assertTrue(res.hits().getError() != null); + assertEquals(Error.TIMEOUT.code, res.hits().getError().getCode()); + assertFalse(err.continuedOnFail); + + err = new ErrorProvider(0); + exec = newExecution(new GroupingExecutor(), + err, + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(grp0), null), + new GroupingListHit(Arrays.asList(grp1), null)))); + res = exec.search(query); + assertTrue(res.hits().getError() != null); + assertEquals(Error.TIMEOUT.code, res.hits().getError().getCode()); + assertFalse(err.continuedOnFail); + } + + @Test + public void requireThatHitsAreFilledWithCorrectSummary() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar))) as(bar) " + + " each(output(summary(baz))) as(baz)))")); + Grouping pass0A = new Grouping(0); + pass0A.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")) + )); + Grouping pass0B = new Grouping(1); + pass0B.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "baz")) + )); + GlobalId gid1 = new GlobalId((new DocumentId("doc:test:1")).getGlobalId()); + GlobalId gid2 = new GlobalId((new DocumentId("doc:test:2")).getGlobalId()); + Grouping pass1A = new Grouping(0); + pass1A.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar").addHit(new com.yahoo.searchlib.aggregation.FS4Hit(1, gid1, 3))) + )); + Grouping pass1B = new Grouping(1); + pass1B.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "baz").addHit(new com.yahoo.searchlib.aggregation.FS4Hit(4, gid2, 6))) + )); + SummaryMapper sm = new SummaryMapper(); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(pass0A, pass0B), null), + new GroupingListHit(Arrays.asList(pass1A, pass1B), null))), + sm); + exec.fill(exec.search(query), "default"); + assertEquals(2, sm.hitsBySummary.size()); + + List<Hit> lst = sm.hitsBySummary.get("bar"); + assertNotNull(lst); + assertEquals(1, lst.size()); + Hit hit = lst.get(0); + assertTrue(hit instanceof FastHit); + assertEquals(1, ((FastHit)hit).getPartId()); + assertEquals(gid1, ((FastHit)hit).getGlobalId()); + + assertNotNull(lst = sm.hitsBySummary.get("baz")); + assertNotNull(lst); + assertEquals(1, lst.size()); + hit = lst.get(0); + assertTrue(hit instanceof FastHit); + assertEquals(4, ((FastHit)hit).getPartId()); + assertEquals(gid2, ((FastHit)hit).getGlobalId()); + } + + @Test + public void requireThatDefaultSummaryNameFillsHitsWithNull() { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary()))) as(foo))")); + + Grouping pass0 = new Grouping(0); + pass0.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("foo")) + .addAggregationResult( + new HitsAggregationResult(1, ExpressionConverter.DEFAULT_SUMMARY_NAME)))); + Grouping pass1 = new Grouping(0); + pass1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("foo")) + .addAggregationResult( + new HitsAggregationResult(1, ExpressionConverter.DEFAULT_SUMMARY_NAME) + .addHit(new com.yahoo.searchlib.aggregation.FS4Hit())))); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList( + new GroupingListHit(Arrays.asList(pass0), null), + new GroupingListHit(Arrays.asList(pass1), null)))); + Result res = exec.search(query); + exec.fill(res); + + Hit hit = ((HitList)((Group)((GroupList)req.getResultGroup(res).get(0)).get(0)).get(0)).get(0); + assertTrue(hit instanceof FastHit); + assertTrue(hit.isFilled(null)); + } + + @Test + public void requireThatHitsAreAttachedToCorrectQuery() { + Query queryA = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(queryA); + req.setRootOperation(GroupingOperation.fromString("all(group(foo) each(each(output(summary(bar)))))")); + + Grouping grp = new Grouping(0); + grp.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar")) + )); + GroupingListHit pass0 = new GroupingListHit(Arrays.asList(grp), null); + + GlobalId gid = new GlobalId((new DocumentId("doc:test:1")).getGlobalId()); + grp = new Grouping(0); + grp.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group().setId(new StringResultNode("foo")) + .addAggregationResult(new HitsAggregationResult(1, "bar").addHit(new com.yahoo.searchlib.aggregation.FS4Hit(4, gid, 6))) + )); + GroupingListHit pass1 = new GroupingListHit(Arrays.asList(grp), null); + Query queryB = newQuery(); /** required by {@link GroupingListHit#getSearchQuery()} */ + pass1.setQuery(queryB); + + QueryMapper qm = new QueryMapper(); + Execution exec = newExecution(new GroupingExecutor(), + new ResultProvider(Arrays.asList(pass0, pass1)), + qm); + exec.fill(exec.search(queryA)); + assertEquals(1, qm.hitsByQuery.size()); + assertTrue(qm.hitsByQuery.containsKey(queryB)); + } + + /** + * Tests the internal rewriting of rank properties which happens in the query.prepare() call + * (triggered by the exc.search call in the below). + */ + @Test + public void testRankProperties() { + Execution exc = newExecution(new GroupingExecutor()); + { + Query query = new Query("?query=foo"); + exc.search(query); + } + { + Query query = new Query("?query=foo&rankfeature.fieldMatch(foo)=2"); + assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + exc.search(query); + assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + } + { + Query query = new Query("?query=foo&rankfeature.query(now)=4"); + assertEquals("4", query.getRanking().getFeatures().get("query(now)")); + exc.search(query); + assertEquals("4", query.getRanking().getProperties().get("now").get(0)); + } + { + Query query = new Query("?query=foo&rankfeature.$bar=8"); + assertEquals("8", query.getRanking().getFeatures().get("$bar")); + exc.search(query); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + } + { + Query query = new Query("?query=foo&rankproperty.bar=8"); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + exc.search(query); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + } + { + Query query = new Query("?query=foo&rankfeature.fieldMatch(foo)=2&rankfeature.query(now)=4&rankproperty.bar=8"); + assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals("4", query.getRanking().getFeatures().get("query(now)")); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + exc.search(query); + assertEquals("2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals("4", query.getRanking().getProperties().get("now").get(0)); + assertEquals("8", query.getRanking().getProperties().get("bar").get(0)); + } + } + + @Test + public void testIllegalQuery() { + Execution exc = newExecution(new GroupingExecutor()); + + Query query = new Query(); + NotItem notItem = new NotItem(); + + notItem.addNegativeItem(new WordItem("negative")); + query.getModel().getQueryTree().setRoot(notItem); + + Result result = exc.search(query); + com.yahoo.search.result.ErrorMessage message = result.hits().getError(); + + assertNotNull("Got error", message); + assertEquals("Illegal query", message.getMessage()); + assertEquals("Can not search for only negative items", + message.getDetailedMessage()); + assertEquals(3, message.getCode()); + } + + // -------------------------------------------------------------------------------- + // + // Utilities + // + // -------------------------------------------------------------------------------- + + private static Query newQuery() { + return new Query("?query=dummy"); + } + + private static Execution newExecution(Searcher... searchers) { + return new Execution(new SearchChain(new ComponentId("foo"), Arrays.asList(searchers)), + Execution.Context.createContextStub()); + } + + private List<Grouping> execute(String groupingExpression) { + Query query = newQuery(); + GroupingRequest req = GroupingRequest.newInstance(query); + req.setRootOperation(GroupingOperation.fromString(groupingExpression)); + GroupingCollector collector = new GroupingCollector(); + newExecution(new GroupingExecutor(), collector).search(query); + return collector.lst; + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class FillRequestThrower extends Searcher { + + @Override + public Result search(Query query, Execution exec) { + return exec.search(query); + } + + @Override + public void fill(Result result, String summaryClass, Execution exec) { + throw new FillRequestException(summaryClass); + } + } + + @SuppressWarnings("serial") + private static class FillRequestException extends RuntimeException { + + final String summaryClass; + + FillRequestException(String summaryClass) { + this.summaryClass = summaryClass; + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class GroupingListThrower extends Searcher { + + @Override + public Result search(Query query, Execution exec) { + throw new GroupingListException(GroupingExecutor.getGroupingList(query)); + } + } + + @SuppressWarnings("serial") + private static class GroupingListException extends RuntimeException { + + final List<Grouping> lst; + + GroupingListException(List<Grouping> lst) { + this.lst = lst; + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class GroupingCollector extends Searcher { + + List<Grouping> lst = new ArrayList<>(); + + @Override + public Result search(Query query, Execution exec) { + for (Grouping grp : GroupingExecutor.getGroupingList(query)) { + lst.add(grp.clone()); + } + return exec.search(query); + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class ErrorProvider extends Searcher { + private final int failOnPassN; + private int passnum; + public boolean continuedOnFail; + + public ErrorProvider(int failOnPassN) { + this.failOnPassN = failOnPassN; + this.passnum = 0; + this.continuedOnFail = false; + } + @Override + public Result search(Query query, Execution exec) { + Result ret = exec.search(query); + if (passnum > failOnPassN) { + continuedOnFail = true; + return ret; + } + if (passnum == failOnPassN) { + ret.hits().setError(ErrorMessage.createTimeout("timeout")); + } + passnum++; + return ret; + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class PassCounter extends Searcher { + + int numPasses = 0; + + @Override + public Result search(Query query, Execution exec) { + ++numPasses; + return exec.search(query); + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class GroupingCounter extends Searcher { + + List<Integer> passList = new ArrayList<>(); + + @Override + public Result search(Query query, Execution exec) { + passList.add(GroupingExecutor.getGroupingList(query).size()); + return exec.search(query); + } + } + + private static class QueryMapper extends Searcher { + + final Map<Query, List<Hit>> hitsByQuery = new HashMap<>(); + + @Override + public Result search(Query query, Execution exec) { + return exec.search(query); + } + + @Override + public void fill(Result result, String summaryClass, Execution exec) { + for (Iterator<Hit> it = result.hits().deepIterator(); it.hasNext();) { + Hit hit = it.next(); + Query query = hit.getQuery(); + List<Hit> lst = hitsByQuery.get(query); + if (lst == null) { + lst = new LinkedList<>(); + hitsByQuery.put(query, lst); + } + lst.add(hit); + } + } + } + + + @After (GroupingExecutor.COMPONENT_NAME) + private static class SummaryMapper extends Searcher { + + final Map<String, List<Hit>> hitsBySummary = new HashMap<>(); + + @Override + public Result search(Query query, Execution exec) { + return exec.search(query); + } + + @Override + public void fill(Result result, String summaryClass, Execution exec) { + for (Iterator<Hit> it = result.hits().deepIterator(); it.hasNext();) { + Hit hit = it.next(); + List<Hit> lst = hitsBySummary.get(summaryClass); + if (lst == null) { + lst = new LinkedList<>(); + hitsBySummary.put(summaryClass, lst); + } + lst.add(hit); + } + } + } + + @After (GroupingExecutor.COMPONENT_NAME) + private static class ResultProvider extends Searcher { + + final Queue<GroupingListHit> hits = new LinkedList<>(); + int pass = 0; + + ResultProvider(List<GroupingListHit> hits) { + this.hits.addAll(hits); + } + + @Override + public Result search(Query query, Execution exec) { + GroupingListHit hit = hits.poll(); + for (Grouping grp : hit.getGroupingList()) { + grp.setFirstLevel(pass); + grp.setLastLevel(pass); + } + ++pass; + Result res = exec.search(query); + res.hits().add(hit); + return res; + } + } + + private static class FillErrorProvider extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + return execution.search(query); + } + + @Override + public void fill(Result result, String summaryClass, Execution exec) { + result.hits().addError(ErrorMessage.createInternalServerError("foo")); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java new file mode 100644 index 00000000000..898a73a3320 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingTransformTestCase.java @@ -0,0 +1,227 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class GroupingTransformTestCase { + + private static final int REQUEST_ID = 0; + + @Test + public void requireThatLabelCanBeSet() { + GroupingTransform transform = newTransform(); + transform.putLabel(0, 1, "foo", "my_type"); + assertEquals("foo", transform.getLabel(1)); + } + + @Test + public void requireThatLabelCanNotBeReplaced() { + GroupingTransform transform = newTransform(); + transform.putLabel(0, 1, "foo", "my_type"); + try { + transform.putLabel(0, 1, "bar", "my_type"); + fail(); + } catch (IllegalStateException e) { + assertEquals("Can not set label of my_type 1 to 'bar' because it is already set to 'foo'.", + e.getMessage()); + } + } + + @Test + public void requireThatLabelIsUniqueAmongSiblings() { + GroupingTransform transform = newTransform(); + transform.putLabel(0, 1, "foo", "my_type"); + try { + transform.putLabel(0, 2, "foo", "my_type"); + fail(); + } catch (UnsupportedOperationException e) { + assertEquals("Can not use my_type label 'foo' for multiple siblings.", + e.getMessage()); + } + } + + @Test + public void requireThatMaxDefaultsToZero() { + GroupingTransform transform = newTransform(); + assertEquals(0, transform.getMax(6)); + assertEquals(0, transform.getMax(9)); + } + + @Test + public void requireThatMaxCanBeSet() { + GroupingTransform transform = newTransform(); + transform.putMax(0, 69, "my_type"); + assertEquals(69, transform.getMax(0)); + } + + @Test + public void requireThatMaxCanNotBeReplaced() { + GroupingTransform transform = newTransform(); + transform.putMax(0, 6, "my_type"); + try { + transform.putMax(0, 9, "my_type"); + fail(); + } catch (IllegalStateException e) { + assertEquals("Can not set max of my_type 0 to 9 because it is already set to 6.", + e.getMessage()); + } + assertEquals(6, transform.getMax(0)); + } + + @Test + public void requireThatOffsetDefaultsToZero() { + GroupingTransform transform = newTransform(); + assertEquals(0, transform.getOffset(6)); + assertEquals(0, transform.getOffset(9)); + } + + @Test + public void requireThatOffsetContinuationsCanBeAdded() { + GroupingTransform transform = newTransform(); + transform.addContinuation(newStableOffset(newResultId(), 6, 9)); + assertEquals(9, transform.getOffset(6)); + } + + @Test + public void requireThatOffsetByIdCanBeReplaced() { + GroupingTransform transform = newTransform(); + ResultId id = newResultId(6, 9); + transform.addContinuation(newStableOffset(id, 0, 6)); + assertEquals(6, transform.getOffset(id)); + transform.addContinuation(newStableOffset(id, 0, 69)); + assertEquals(69, transform.getOffset(id)); + transform.addContinuation(newStableOffset(id, 0, 9)); + assertEquals(9, transform.getOffset(id)); + transform.addContinuation(newStableOffset(id, 0, 96)); + assertEquals(96, transform.getOffset(id)); + } + + @Test + public void requireThatOffsetByTagEqualsHighestSibling() { + GroupingTransform transform = newTransform(); + transform.addContinuation(newStableOffset(newResultId(1), 69, 6)); + assertEquals(6, transform.getOffset(69)); + transform.addContinuation(newStableOffset(newResultId(2), 69, 69)); + assertEquals(69, transform.getOffset(69)); + transform.addContinuation(newStableOffset(newResultId(3), 69, 9)); + assertEquals(69, transform.getOffset(69)); + transform.addContinuation(newStableOffset(newResultId(4), 69, 96)); + assertEquals(96, transform.getOffset(69)); + } + + @Test + public void requireThatOffsetContinuationsCanBeReplaced() { + GroupingTransform transform = newTransform(); + ResultId id = newResultId(6, 9); + transform.addContinuation(newStableOffset(id, 1, 1)); + assertEquals(1, transform.getOffset(1)); + assertEquals(1, transform.getOffset(id)); + assertTrue(transform.isStable(id)); + + transform.addContinuation(newUnstableOffset(id, 1, 2)); + assertEquals(2, transform.getOffset(1)); + assertEquals(2, transform.getOffset(id)); + assertFalse(transform.isStable(id)); + + transform.addContinuation(newStableOffset(id, 1, 3)); + assertEquals(3, transform.getOffset(1)); + assertEquals(3, transform.getOffset(id)); + assertTrue(transform.isStable(id)); + } + + @Test + public void requireThatUnstableOffsetsAreTracked() { + GroupingTransform transform = newTransform(); + ResultId stableId = newResultId(6); + transform.addContinuation(newStableOffset(stableId, 1, 1)); + assertTrue(transform.isStable(stableId)); + ResultId unstableId = newResultId(9); + transform.addContinuation(newUnstableOffset(unstableId, 2, 3)); + assertTrue(transform.isStable(stableId)); + assertFalse(transform.isStable(unstableId)); + } + + @Test + public void requireThatCompositeContinuationsAreDecomposed() { + GroupingTransform transform = newTransform(); + transform.addContinuation(new CompositeContinuation() + .add(newStableOffset(newResultId(), 6, 9)) + .add(newStableOffset(newResultId(), 9, 6))); + assertEquals(9, transform.getOffset(6)); + assertEquals(6, transform.getOffset(9)); + } + + @Test + public void requireThatUnsupportedContinuationsCanNotBeAdded() { + GroupingTransform transform = newTransform(); + try { + transform.addContinuation(new Continuation() { + + }); + fail(); + } catch (UnsupportedOperationException e) { + + } + } + + @Test + public void requireThatUnrelatedContinuationsAreIgnored() { + GroupingTransform transform = new GroupingTransform(REQUEST_ID); + ResultId id = ResultId.valueOf(REQUEST_ID + 1, 1); + transform.addContinuation(new OffsetContinuation(id, 2, 3, OffsetContinuation.FLAG_UNSTABLE)); + assertEquals(0, transform.getOffset(2)); + assertEquals(0, transform.getOffset(id)); + assertTrue(transform.isStable(id)); + } + + @Test + public void requireThatToStringIsVerbose() { + GroupingTransform transform = new GroupingTransform(REQUEST_ID); + transform.putLabel(1, 1, "label1", "type1"); + transform.putLabel(2, 2, "label2", "type2"); + transform.addContinuation(newStableOffset(ResultId.valueOf(REQUEST_ID), 3, 3)); + transform.addContinuation(newStableOffset(ResultId.valueOf(REQUEST_ID), 4, 4)); + transform.putMax(5, 5, "type5"); + transform.putMax(6, 6, "type6"); + assertEquals("groupingTransform {\n" + + "\tlabels {\n" + + "\t\t1 : label1\n" + + "\t\t2 : label2\n" + + "\t}\n" + + "\toffsets {\n" + + "\t\t3 : 3\n" + + "\t\t4 : 4\n" + + "\t}\n" + + "\tmaxes {\n" + + "\t\t5 : 5\n" + + "\t\t6 : 6\n" + + "\t}\n" + + "}", transform.toString()); + } + + private static GroupingTransform newTransform() { + return new GroupingTransform(REQUEST_ID); + } + + private static ResultId newResultId(int... indexes) { + ResultId id = ResultId.valueOf(REQUEST_ID); + for (int i : indexes) { + id = id.newChildId(i); + } + return id; + } + + private static OffsetContinuation newStableOffset(ResultId resultId, int tag, int offset) { + return new OffsetContinuation(resultId, tag, offset, 0); + } + + private static OffsetContinuation newUnstableOffset(ResultId resultId, int tag, int offset) { + return new OffsetContinuation(resultId, tag, offset, OffsetContinuation.FLAG_UNSTABLE); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java new file mode 100644 index 00000000000..ebd663d80b0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/HitConverterTestCase.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.GlobalId; +import com.yahoo.fs4.QueryPacketData; +import com.yahoo.net.URI; +import com.yahoo.prelude.fastsearch.GroupingListHit; +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.searchlib.aggregation.FS4Hit; +import com.yahoo.searchlib.aggregation.VdsHit; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class HitConverterTestCase { + + private GlobalId createGlobalId(int docId) { + return new GlobalId((new DocumentId("doc:test:" + docId)).getGlobalId()); + } + + @Test + public void requireThatHitsAreConverted() { + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + Hit hit = converter.toSearchHit("default", new FS4Hit(1, createGlobalId(2), 3).setContext(new Hit("hit:ctx"))); + assertNotNull(hit); + assertEquals(new URI("index:0/1/0/" + FastHit.asHexString(createGlobalId(2))), hit.getId()); + + hit = converter.toSearchHit("default", new FS4Hit(4, createGlobalId(5), 6).setContext(new Hit("hit:ctx"))); + assertNotNull(hit); + assertEquals(new URI("index:0/4/0/" + FastHit.asHexString(createGlobalId(5))), hit.getId()); + } + + @Test + public void requireThatContextDataIsCopied() { + Hit ctxHit = new Hit("hit:ctx"); + ctxHit.setSource("69"); + ctxHit.setSourceNumber(69); + Query ctxQuery = new Query(); + ctxHit.setQuery(ctxQuery); + + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + Hit hit = converter.toSearchHit("default", new FS4Hit(1, createGlobalId(2), 3).setContext(ctxHit)); + assertNotNull(hit); + assertTrue(hit instanceof FastHit); + assertEquals(1, ((FastHit)hit).getPartId()); + assertEquals(createGlobalId(2), ((FastHit)hit).getGlobalId()); + assertSame(ctxQuery, hit.getQuery()); + assertEquals(ctxHit.getSource(), hit.getSource()); + assertEquals(ctxHit.getSourceNumber(), hit.getSourceNumber()); + } + + @Test + public void requireThatHitTagIsCopiedFromGroupingListContext() { + QueryPacketData ctxTag = new QueryPacketData(); + GroupingListHit ctxHit = new GroupingListHit(null, null); + ctxHit.setQueryPacketData(ctxTag); + + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + Hit hit = converter.toSearchHit("default", new FS4Hit(1, createGlobalId(2), 3).setContext(ctxHit)); + assertNotNull(hit); + assertTrue(hit instanceof FastHit); + assertSame(ctxTag, ((FastHit)hit).getQueryPacketData()); + } + + @Test + public void requireThatSummaryClassIsSet() { + Searcher searcher = new MySearcher(); + HitConverter converter = new HitConverter(searcher, new Query()); + Hit hit = converter.toSearchHit("69", new FS4Hit(1, createGlobalId(2), 3).setContext(new Hit("hit:ctx"))); + assertNotNull(hit); + assertTrue(hit instanceof FastHit); + assertEquals("69", hit.getSearcherSpecificMetaData(searcher)); + } + + @Test + public void requireThatHitHasContext() { + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + try { + converter.toSearchHit("69", new FS4Hit(1, createGlobalId(2), 3)); + fail(); + } catch (NullPointerException e) { + + } + } + + @Test + public void requireThatUnsupportedHitClassThrows() { + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + try { + converter.toSearchHit("69", new com.yahoo.searchlib.aggregation.Hit() { + + }); + fail(); + } catch (UnsupportedOperationException e) { + + } + } + + private static DocumentdbInfoConfig.Documentdb sixtynine() { + DocumentdbInfoConfig.Documentdb.Builder summaryConfig = new DocumentdbInfoConfig.Documentdb.Builder(); + summaryConfig.name("none"); + summaryConfig.summaryclass(new DocumentdbInfoConfig.Documentdb.Summaryclass.Builder().id(0).name("69")); + return new DocumentdbInfoConfig.Documentdb(summaryConfig); + } + + @Test + public void requireThatVdsHitCanBeConverted() { + HitConverter converter = new HitConverter(new MySearcher(), new Query()); + GroupingListHit context = new GroupingListHit(null, new DocsumDefinitionSet(sixtynine())); + VdsHit lowHit = new VdsHit("doc:scheme:", new byte[] { 0, 0, 0, 0 }, 1); + lowHit.setContext(context); + Hit hit = converter.toSearchHit("69", lowHit); + assertNotNull(hit); + assertTrue(hit instanceof FastHit); + assertEquals(new Relevance(1), hit.getRelevance()); + assertTrue(hit.isFilled("69")); + } + + private static class MySearcher extends Searcher { + + @Override + public Result search(Query query, Execution exec) { + return exec.search(query); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerDecoderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerDecoderTestCase.java new file mode 100644 index 00000000000..9389482010e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerDecoderTestCase.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import org.junit.Test; + +import static org.junit.Assert.*; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IntegerDecoderTestCase { + + @Test + public void requireThatIntDecoderWorksAsExpected() { + assertDecode("A", 0); + assertDecode("BC", 1); + assertDecode("CBI", 12); + assertDecode("CPG", 123); + assertDecode("DJKE", 1234); + assertDecode("EGAHC", 12345); + assertDecode("FDMEIA", 123456); + assertDecode("GCFKNAO", 1234567); + assertDecode("HBHIMCJM", 12345678); + assertDecode("HOLHJKCK", 123456789); + assertDecode("IJDCMAFKE", 1234567890); + assertDecode("IIKKEBPOF", -1163005939); + assertDecode("IECKEIKID", -559039810); + } + + @Test + public void requireThatDecoderThrowsExceptionOnBadInput() { + try { + new IntegerDecoder("B").next(); + fail(); + } catch (IndexOutOfBoundsException e) { + + } + try { + new IntegerDecoder("11X1Y").next(); + fail(); + } catch (NumberFormatException e) { + + } + } + + private static void assertDecode(String toDecode, int expected) { + IntegerDecoder decoder = new IntegerDecoder(toDecode); + assertTrue(decoder.hasNext()); + assertEquals(expected, decoder.next()); + assertFalse(decoder.hasNext()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerEncoderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerEncoderTestCase.java new file mode 100644 index 00000000000..4780f23ca9d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/IntegerEncoderTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IntegerEncoderTestCase { + + @Test + public void requireThatIntEncoderWorksAsExpected() { + assertEncode("A", 0); + assertEncode("BC", 1); + assertEncode("CBI", 12); + assertEncode("CPG", 123); + assertEncode("DJKE", 1234); + assertEncode("EGAHC", 12345); + assertEncode("FDMEIA", 123456); + assertEncode("GCFKNAO", 1234567); + assertEncode("HBHIMCJM", 12345678); + assertEncode("HOLHJKCK", 123456789); + assertEncode("IJDCMAFKE", 1234567890); + assertEncode("IIKKEBPOF", -1163005939); + assertEncode("IECKEIKID", -559039810); + } + + private static void assertEncode(String expected, int toEncode) { + IntegerEncoder actual = new IntegerEncoder(); + actual.append(toEncode); + assertEquals(expected, actual.toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/OffsetContinuationTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/OffsetContinuationTestCase.java new file mode 100644 index 00000000000..8184a52c0ee --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/OffsetContinuationTestCase.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.search.grouping.Continuation; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class OffsetContinuationTestCase { + + @Test + public void requireThatNullResultIdThrowsException() { + try { + new OffsetContinuation(null, 0, 0, 0); + fail(); + } catch (NullPointerException e) { + + } + } + + @Test + public void requireThatAccessorsWork() { + OffsetContinuation cnt = new OffsetContinuation(ResultId.valueOf(1), 2, 3, 4); + assertEquals(ResultId.valueOf(1), cnt.getResultId()); + assertEquals(2, cnt.getTag()); + assertEquals(3, cnt.getOffset()); + assertEquals(4, cnt.getFlags()); + + cnt = new OffsetContinuation(ResultId.valueOf(5), 6, 7, 8); + assertEquals(ResultId.valueOf(5), cnt.getResultId()); + assertEquals(6, cnt.getTag()); + assertEquals(7, cnt.getOffset()); + assertEquals(8, cnt.getFlags()); + + for (int i = 0; i < 30; ++i) { + cnt = new OffsetContinuation(ResultId.valueOf(1), 2, 3, (1 << i) + (1 << i + 1)); + assertTrue(cnt.testFlag(1 << i)); + assertTrue(cnt.testFlag(1 << i + 1)); + assertFalse(cnt.testFlag(1 << i + 2)); + } + } + + @Test + public void requireThatOffsetContinuationsCanBeEncoded() { + assertEncode("BCBCBCBEBG", newOffset(1, 1, 2, 3)); + assertEncode("BCBKCBACBKCCK", newOffset(5, 8, 13, 21)); + assertEncode("BCBBBBBDBF", newOffset(-1, -1, -2, -3)); + assertEncode("BCBJBPCBJCCJ", newOffset(-5, -8, -13, -21)); + } + + @Test + public void requireThatOffsetContinuationsCanBeDecoded() { + assertDecode("BCBCBCBEBG", newOffset(1, 1, 2, 3)); + assertDecode("BCBKCBACBKCCK", newOffset(5, 8, 13, 21)); + assertDecode("BCBBBBBDBF", newOffset(-1, -1, -2, -3)); + assertDecode("BCBJBPCBJCCJ", newOffset(-5, -8, -13, -21)); + } + + @Test + public void requireThatHashCodeIsImplemented() { + assertEquals(newOffset(1, 1, 2, 3).hashCode(), newOffset(1, 1, 2, 3).hashCode()); + } + + @Test + public void requireThatEqualsIsImplemented() { + Continuation cnt = newOffset(1, 1, 2, 3); + assertFalse(cnt.equals(new Object())); + assertFalse(cnt.equals(newOffset(0, 1, 2, 3))); + assertFalse(cnt.equals(newOffset(1, 0, 2, 3))); + assertFalse(cnt.equals(newOffset(1, 1, 0, 3))); + assertFalse(cnt.equals(newOffset(1, 1, 2, 0))); + assertEquals(cnt, newOffset(1, 1, 2, 3)); + } + + + private static OffsetContinuation newOffset(int resultId, int tag, int offset, int flags) { + return new OffsetContinuation(ResultId.valueOf(resultId), tag, offset, flags); + } + + private static void assertEncode(String expected, EncodableContinuation toEncode) { + IntegerEncoder actual = new IntegerEncoder(); + toEncode.encode(actual); + assertEquals(expected, actual.toString()); + } + + private static void assertDecode(String toDecode, Continuation expected) { + assertEquals(expected, OffsetContinuation.decode(new IntegerDecoder(toDecode))); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java new file mode 100644 index 00000000000..9fd32577737 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/RequestBuilderTestCase.java @@ -0,0 +1,885 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.grouping.request.*; +import com.yahoo.searchlib.aggregation.*; +import com.yahoo.searchlib.expression.*; +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class RequestBuilderTestCase { + + @Test + public void requireThatAllAggregationResulsAreSupported() { + assertLayout("all(group(a) each(output(avg(b))))", "[[{ Attribute, result = [Average] }]]"); + assertLayout("all(group(a) each(output(count())))", "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(group(a) each(output(max(b))))", "[[{ Attribute, result = [Max] }]]"); + assertLayout("all(group(a) each(output(min(b))))", "[[{ Attribute, result = [Min] }]]"); + assertLayout("all(group(a) each(output(sum(b))))", "[[{ Attribute, result = [Sum] }]]"); + assertLayout("all(group(a) each(each(output(summary()))))", "[[{ Attribute, result = [Hits] }]]"); + assertLayout("all(group(a) each(output(xor(b))))", "[[{ Attribute, result = [Xor] }]]"); + } + + @Test + public void requireThatExpressionCountAggregationResultIsSupported() { + RequestBuilder builder = new RequestBuilder(0); + builder.setRootOperation(GroupingOperation.fromString("all(group(foo) output(count()))")); + builder.build(); + AggregationResult aggr = builder.getRequestList().get(0).getRoot().getAggregationResults().get(0); + assertTrue(aggr instanceof ExpressionCountAggregationResult); + assertEquals(new AttributeNode("foo"), aggr.getExpression()); + } + + @Test + public void requireThatAllExpressionNodesAreSupported() { + assertLayout("all(group(add(a,b)) each(output(count())))", "[[{ Add, result = [Count] }]]"); + assertLayout("all(group(and(a,b)) each(output(count())))", "[[{ And, result = [Count] }]]"); + assertLayout("all(group(a) each(output(count())))", "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(group(cat(a,b)) each(output(count())))", "[[{ Cat, result = [Count] }]]"); + assertLayout("all(group(debugwait(a, 69, true)) each(output(count())))", "[[{ DebugWait, result = [Count] }]]"); + assertLayout("all(group(docidnsspecific()) each(output(count())))", "[[{ GetDocIdNamespaceSpecific, result = [Count] }]]"); + assertLayout("all(group(1.0) each(output(count())))", "[[{ Constant, result = [Count] }]]"); + assertLayout("all(group(div(a,b)) each(output(count())))", "[[{ Divide, result = [Count] }]]"); + assertLayout("all(group(fixedwidth(a,1)) each(output(count())))", "[[{ FixedWidthBucket, result = [Count] }]]"); + assertLayout("all(group(fixedwidth(a,1.0)) each(output(count())))", "[[{ FixedWidthBucket, result = [Count] }]]"); + assertLayout("all(group(1) each(output(count())))", "[[{ Constant, result = [Count] }]]"); + assertLayout("all(group(max(a,b)) each(output(count())))", "[[{ Max, result = [Count] }]]"); + assertLayout("all(group(md5(a,1)) each(output(count())))", "[[{ MD5Bit, result = [Count] }]]"); + assertLayout("all(group(uca(a,b)) each(output(count())))", "[[{ Uca, result = [Count] }]]"); + assertLayout("all(group(uca(a,b,PRIMARY)) each(output(count())))", "[[{ Uca, result = [Count] }]]"); + assertLayout("all(group(min(a,b)) each(output(count())))", "[[{ Min, result = [Count] }]]"); + assertLayout("all(group(mod(a,b)) each(output(count())))", "[[{ Modulo, result = [Count] }]]"); + assertLayout("all(group(mul(a,b)) each(output(count())))", "[[{ Multiply, result = [Count] }]]"); + assertLayout("all(group(neg(a)) each(output(count())))", "[[{ Negate, result = [Count] }]]"); + assertLayout("all(group(normalizesubject(a)) each(output(count())))", "[[{ NormalizeSubject, result = [Count] }]]"); + assertLayout("all(group(now()) each(output(count())))", "[[{ Constant, result = [Count] }]]"); + assertLayout("all(group(or(a,b)) each(output(count())))", "[[{ Or, result = [Count] }]]"); + assertLayout("all(group(predefined(a,bucket(1,2))) each(output(count())))", "[[{ RangeBucketPreDef, result = [Count] }]]"); + assertLayout("all(group(relevance()) each(output(count())))", "[[{ Relevance, result = [Count] }]]"); + assertLayout("all(group(reverse(a)) each(output(count())))", "[[{ Reverse, result = [Count] }]]"); + assertLayout("all(group(size(a)) each(output(count())))", "[[{ NumElem, result = [Count] }]]"); + assertLayout("all(group(sort(a)) each(output(count())))", "[[{ Sort, result = [Count] }]]"); + assertLayout("all(group(strcat(a,b)) each(output(count())))", "[[{ StrCat, result = [Count] }]]"); + assertLayout("all(group('a') each(output(count())))", "[[{ Constant, result = [Count] }]]"); + assertLayout("all(group(strlen(a)) each(output(count())))", "[[{ StrLen, result = [Count] }]]"); + assertLayout("all(group(sub(a,b)) each(output(count())))", "[[{ Add, result = [Count] }]]"); + assertLayout("all(group(todouble(a)) each(output(count())))", "[[{ ToFloat, result = [Count] }]]"); + assertLayout("all(group(tolong(a)) each(output(count())))", "[[{ ToInt, result = [Count] }]]"); + assertLayout("all(group(toraw(a)) each(output(count())))", "[[{ ToRaw, result = [Count] }]]"); + assertLayout("all(group(tostring(a)) each(output(count())))", "[[{ ToString, result = [Count] }]]"); + assertLayout("all(group(time.date(a)) each(output(count())))", "[[{ StrCat, result = [Count] }]]"); + assertLayout("all(group(math.sqrt(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.cbrt(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.log(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.log1p(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.log10(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.exp(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.pow(a,b)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.hypot(a,b)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.sin(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.asin(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.cos(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.acos(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.tan(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.atan(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.sinh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.asinh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.cosh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.acosh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(math.tanh(a)) each(output(count())))", "[[{ Math, result = [Count] }]]"); + assertLayout("all(group(zcurve.x(a)) each(output(count())))", "[[{ ZCurve, result = [Count] }]]"); + assertLayout("all(group(zcurve.y(a)) each(output(count())))", "[[{ ZCurve, result = [Count] }]]"); + assertLayout("all(group(time.dayofmonth(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.dayofweek(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.dayofyear(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.hourofday(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.minuteofhour(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.monthofyear(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.secondofminute(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(time.year(a)) each(output(count())))", "[[{ TimeStamp, result = [Count] }]]"); + assertLayout("all(group(xor(a,b)) each(output(count())))", "[[{ Xor, result = [Count] }]]"); + assertLayout("all(group(xorbit(a,1)) each(output(count())))", "[[{ XorBit, result = [Count] }]]"); + assertLayout("all(group(ymum()) each(output(count())))", "[[{ GetYMUMChecksum, result = [Count] }]]"); + } + + @Test + public void requireThatForceSinglePassIsSupported() { + assertForceSinglePass("all(group(foo) each(output(count())))", "[false]"); + assertForceSinglePass("all(group(foo) hint(singlepass) each(output(count())))", "[true]"); + assertForceSinglePass("all(hint(singlepass) " + + " all(group(foo) each(output(count())))" + + " all(group(bar) each(output(count()))))", + "[true, true]"); + + // it would be really nice if this test returned [true, true], but that is not how the AST is built + assertForceSinglePass("all(all(group(foo) hint(singlepass) each(output(count())))" + + " all(group(bar) hint(singlepass) each(output(count()))))", + "[false, false]"); + } + + @Test + public void requireThatThereCanBeOnlyOneBuildCall() { + RequestBuilder builder = new RequestBuilder(0); + builder.setRootOperation(GroupingOperation.fromString("all(group(foo) each(output(count())))")); + builder.build(); + try { + builder.build(); + fail(); + } catch (IllegalStateException e) { + + } + } + + @Test + public void requireThatNullSummaryClassProvidesDefault() { + RequestBuilder reqBuilder = new RequestBuilder(0); + reqBuilder.setRootOperation(new AllOperation() + .setGroupBy(new AttributeValue("foo")) + .addChild(new EachOperation() + .addChild(new EachOperation() + .addOutput(new SummaryValue())))); + reqBuilder.setDefaultSummaryName(null); + reqBuilder.build(); + + HitsAggregationResult hits = (HitsAggregationResult)reqBuilder.getRequestList().get(0) + .getLevels().get(0) + .getGroupPrototype() + .getAggregationResults().get(0); + assertEquals(ExpressionConverter.DEFAULT_SUMMARY_NAME, hits.getSummaryClass()); + } + + @Test + public void requireThatGroupOfGroupsAreNotSupported() { + // "Can not group list of groups." + assertBuildFail("all(group(a) all(group(avg(b)) each(each(each(output(summary()))))))", + "Can not operate on list of list of groups."); + } + + @Test + public void requireThatAnonymousListsAreNotSupported() { + assertBuildFail("all(group(a) all(each(each(output(summary())))))", + "Can not create anonymous list of groups."); + } + + @Test + public void requireThatOffsetContinuationCanModifyGroupingLevel() { + assertOffset("all(group(a) max(5) each(output(count())))", + newOffset(2, 5), + "[[{ tag = 2, max = [5, 11], hits = [] }]]"); + assertOffset("all(group(a) max(5) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + newOffset(2, 5), + "[[{ tag = 2, max = [5, 11], hits = [] }]," + + " [{ tag = 4, max = [5, 6], hits = [] }]]"); + assertOffset("all(group(a) max(5) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + newComposite(newOffset(2, 5), newOffset(4, 10)), + "[[{ tag = 2, max = [5, 11], hits = [] }]," + + " [{ tag = 4, max = [5, 16], hits = [] }]]"); + } + + @Test + public void requireThatOffsetContinuationCanModifyHitAggregator() { + assertOffset("all(group(a) each(max(5) each(output(summary()))))", + newOffset(3, 5), + "[[{ tag = 2, max = [0, -1], hits = [{ tag = 3, max = [5, 11] }] }]]"); + assertOffset("all(group(a) each(max(5) each(output(summary()))) as(foo)" + + " each(max(5) each(output(summary()))) as(bar))", + newOffset(3, 5), + "[[{ tag = 2, max = [0, -1], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 4, max = [0, -1], hits = [{ tag = 5, max = [5, 6] }] }]]"); + assertOffset("all(group(a) each(max(5) each(output(summary()))) as(foo)" + + " each(max(5) each(output(summary()))) as(bar))", + newComposite(newOffset(3, 5), newOffset(5, 10)), + "[[{ tag = 2, max = [0, -1], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 4, max = [0, -1], hits = [{ tag = 5, max = [5, 16] }] }]]"); + } + + @Test + public void requireThatOffsetContinuationIsNotAppliedToGroupingLevelWithoutMax() { + assertOffset("all(group(a) each(output(count())))", + newOffset(2, 5), + "[[{ tag = 2, max = [0, -1], hits = [] }]]"); + } + + @Test + public void requireThatOffsetContinuationIsNotAppliedToHitAggregatorWithoutMax() { + assertOffset("all(group(a) each(each(output(summary()))))", + newOffset(3, 5), + "[[{ tag = 2, max = [0, -1], hits = [{ tag = 3, max = [0, -1] }] }]]"); + } + + @Test + public void requireThatUnstableContinuationsDoNotAffectRequestedGroupLists() { + String request = "all(group(a) max(5) each(group(b) max(5) each(output(count())) as(a1_b1)" + + " each(output(count())) as(a1_b2)) as(a1)" + + " each(group(b) max(5) each(output(count())) as(a2_b1)" + + " each(output(count())) as(a2_b2)) as(a2))"; + CompositeContinuation session = newComposite(newOffset(2, 5), newOffset(3, 5), newOffset(5, 5), + newOffset(7, 5), newOffset(8, 5), newOffset(10, 5)); + assertOffset(request, newComposite(session), + "[[{ tag = 2, max = [5, 11], hits = [] }, { tag = 3, max = [5, 11], hits = [] }]," + + " [{ tag = 2, max = [5, 11], hits = [] }, { tag = 5, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 11], hits = [] }, { tag = 10, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 11], hits = [] }, { tag = 8, max = [5, 11], hits = [] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(2, 10)), + "[[{ tag = 2, max = [5, 16], hits = [] }, { tag = 3, max = [5, 11], hits = [] }]," + + " [{ tag = 2, max = [5, 16], hits = [] }, { tag = 5, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 11], hits = [] }, { tag = 10, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 11], hits = [] }, { tag = 8, max = [5, 11], hits = [] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(7, 10)), + "[[{ tag = 2, max = [5, 11], hits = [] }, { tag = 3, max = [5, 11], hits = [] }]," + + " [{ tag = 2, max = [5, 11], hits = [] }, { tag = 5, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 16], hits = [] }, { tag = 10, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 16], hits = [] }, { tag = 8, max = [5, 11], hits = [] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(2, 10), newUnstableOffset(7, 10)), + "[[{ tag = 2, max = [5, 16], hits = [] }, { tag = 3, max = [5, 11], hits = [] }]," + + " [{ tag = 2, max = [5, 16], hits = [] }, { tag = 5, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 16], hits = [] }, { tag = 10, max = [5, 11], hits = [] }]," + + " [{ tag = 7, max = [5, 16], hits = [] }, { tag = 8, max = [5, 11], hits = [] }]]"); + } + + @Test + public void requireThatUnstableContinuationsDoNotAffectRequestedHitLists() { + String request = "all(group(a) max(5) each(max(5) each(output(summary())) as(a1_h1)" + + " each(output(summary())) as(a1_h2)) as(a1)" + + " each(max(5) each(output(summary())) as(a2_h1)" + + " each(output(summary())) as(a2_h2)) as(a2))"; + CompositeContinuation session = newComposite(newOffset(2, 5), newOffset(3, 5), newOffset(4, 5), + newOffset(5, 5), newOffset(6, 5), newOffset(7, 5)); + assertOffset(request, newComposite(session), + "[[{ tag = 2, max = [5, 11], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 2, max = [5, 11], hits = [{ tag = 4, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 11], hits = [{ tag = 6, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 11], hits = [{ tag = 7, max = [5, 11] }] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(2, 10)), + "[[{ tag = 2, max = [5, 16], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 2, max = [5, 16], hits = [{ tag = 4, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 11], hits = [{ tag = 6, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 11], hits = [{ tag = 7, max = [5, 11] }] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(5, 10)), + "[[{ tag = 2, max = [5, 11], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 2, max = [5, 11], hits = [{ tag = 4, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 16], hits = [{ tag = 6, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 16], hits = [{ tag = 7, max = [5, 11] }] }]]"); + assertOffset(request, newComposite(session, newUnstableOffset(2, 10), newUnstableOffset(5, 10)), + "[[{ tag = 2, max = [5, 16], hits = [{ tag = 3, max = [5, 11] }] }]," + + " [{ tag = 2, max = [5, 16], hits = [{ tag = 4, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 16], hits = [{ tag = 6, max = [5, 11] }] }]," + + " [{ tag = 5, max = [5, 16], hits = [{ tag = 7, max = [5, 11] }] }]]"); + } + + @Test + public void requireThatExpressionsCanBeAliased() { + OutputWriter writer = (groupingList, transform) -> groupingList.get(0).getLevels().get(0).getGroupPrototype().getAggregationResults().get(0) + .toString(); + + RequestTest test = new RequestTest(); + test.expectedOutput = new SumAggregationResult().setTag(3).setExpression(new AttributeNode("price")).toString(); + test.request = "all(group(artist) alias(foo,sum(price)) each(output($foo)))"; + test.outputWriter = writer; + assertOutput(test); + + test = new RequestTest(); + test.expectedOutput = new SumAggregationResult().setTag(3).setExpression(new AttributeNode("price")).toString(); + test.request = "all(group(artist) order($foo=sum(price)) each(output($foo)))"; + test.outputWriter = writer; + assertOutput(test); + } + + @Test + public void requireThatGroupingLayoutIsCorrect() { + assertLayout("all(group(artist) each(max(69) output(count()) each(output(summary()))))", + "[[{ Attribute, result = [Count, Hits] }]]"); + assertLayout("all(group(artist) each(output(count()) all(group(album) each(output(count()) all(group(song) each(max(69) output(count()) each(output(summary()))))))))", + "[[{ Attribute, result = [Count] }, { Attribute, result = [Count] }, { Attribute, result = [Count, Hits] }]]"); + assertLayout("all(group(artist) each(output(count())))", + "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(group(artist) order(sum(price)) each(output(count())))", + "[[{ Attribute, result = [Count, Sum], order = [[1], [AggregationRef]] }]]"); + assertLayout("all(group(artist) each(max(69) output(count()) each(output(summary(foo)))))", + "[[{ Attribute, result = [Count, Hits] }]]"); + assertLayout("all(group(artist) each(output(count()) all(group(album) each(output(count())))))", + "[[{ Attribute, result = [Count] }, { Attribute, result = [Count] }]]"); + assertLayout("all(group(artist) max(5) each(output(count()) all(group(album) max(3) each(output(count())))))", + "[[{ Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [4, 4], result = [Count] }]]"); + assertLayout("all(group(artist) max(5) each(output(count()) all(group(album) max(3) each(output(count())))))", + "[[{ Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [4, 4], result = [Count] }]]"); + assertLayout("all(group(foo) max(10) each(output(count()) all(group(bar) max(10) each(output(count())))))", + "[[{ Attribute, max = [11, 11], result = [Count] }, { Attribute, max = [11, 11], result = [Count] }]]"); + assertLayout("all(group(a) max(5) each(max(69) output(count()) each(output(summary()))))", + "[[{ Attribute, max = [6, 6], result = [Count, Hits] }]]"); + assertLayout("all(group(a) max(5) each(output(count()) all(group(b) max(5) each(max(69) output(count()) each(output(summary()))))))", + "[[{ Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [6, 6], result = [Count, Hits] }]]"); + assertLayout("all(group(a) max(5) each(output(count()) all(group(b) max(5) each(output(count()) all(group(c) max(5) each(max(69) output(count()) each(output(summary()))))))))", + "[[{ Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [6, 6], result = [Count] }, { Attribute, max = [6, 6], result = [Count, Hits] }]]"); + assertLayout("all(group(fixedwidth(n,3)) max(5) each(output(count()) all(group(a) max(2) each(output(count())))))", + "[[{ FixedWidthBucket, max = [6, 6], result = [Count] }, { Attribute, max = [3, 3], result = [Count] }]]"); + assertLayout("all(group(fixedwidth(n,3)) max(5) each(output(count()) all(group(a) max(2) each(output(count())))))", + "[[{ FixedWidthBucket, max = [6, 6], result = [Count] }, { Attribute, max = [3, 3], result = [Count] }]]"); + assertLayout("all(group(fixedwidth(n,3)) max(5) each(output(count()) all(group(a) max(2) each(max(1) output(count()) each(output(summary()))))))", + "[[{ FixedWidthBucket, max = [6, 6], result = [Count] }, { Attribute, max = [3, 3], result = [Count, Hits] }]]"); + assertLayout("all(group(predefined(n,bucket(1,3),bucket(6,9))) each(output(count())))", + "[[{ RangeBucketPreDef, result = [Count] }]]"); + assertLayout("all(group(predefined(f,bucket(1.0,3.0),bucket(6.0,9.0))) each(output(count())))", + "[[{ RangeBucketPreDef, result = [Count] }]]"); + assertLayout("all(group(predefined(s,bucket(\"ab\",\"cd\"),bucket(\"ef\",\"gh\"))) each(output(count())))", + "[[{ RangeBucketPreDef, result = [Count] }]]"); + assertLayout("all(group(a) max(5) each(output(count())))", + "[[{ Attribute, max = [6, 6], result = [Count] }]]"); + assertLayout("all(group(a) max(5) each(output(count())))", + "[[{ Attribute, max = [6, 6], result = [Count] }]]"); + assertLayout("all(max(9) all(group(a) each(output(count()))))", + "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(where(true) all(group(a) each(output(count()))))", + "[[{ Attribute, result = [Count] }]]"); + assertLayout("all(group(a) order(sum(n)) each(output(count())))", + "[[{ Attribute, result = [Count, Sum], order = [[1], [AggregationRef]] }]]"); + assertLayout("all(group(a) max(2) each(output(count())))", + "[[{ Attribute, max = [3, 3], result = [Count] }]]"); + assertLayout("all(group(a) max(2) precision(10) each(output(count())))", + "[[{ Attribute, max = [3, 10], result = [Count] }]]"); + assertLayout("all(group(fixedwidth(a,1)) each(output(count())))", + "[[{ FixedWidthBucket, result = [Count] }]]"); + } + + @Test + public void requireThatAggregatorCanBeUsedAsArgumentToOrderByFunction() { + assertLayout("all(group(a) order(sum(price) * count()) each(output(count())))", + "[[{ Attribute, result = [Count, Sum], order = [[1], [Multiply]] }]]"); + assertLayout("all(group(a) order(sum(price) + 4) each(output(sum(price))))", + "[[{ Attribute, result = [Sum], order = [[1], [Add]] }]]"); + assertLayout("all(group(a) order(sum(price) + 4, count()) each(output(sum(price))))", + "[[{ Attribute, result = [Sum, Count], order = [[1, 2], [Add, AggregationRef]] }]]"); + assertLayout("all(group(a) order(sum(price) + 4, -count()) each(output(sum(price))))", + "[[{ Attribute, result = [Sum, Count], order = [[1, -2], [Add, AggregationRef]] }]]"); + } + + @Test + public void requireThatSameAggregatorCanBeUsedMultipleTimes() { + assertLayout("all(group(a) each(output(count() as(b),count() as(c))))", + "[[{ Attribute, result = [Count, Count] }]]"); + } + + @Test + public void requireThatSiblingAggregatorsCanNotShareSameLabel() { + assertBuildFail("all(group(a) each(output(count(),count())))", + "Can not use output label 'count()' for multiple siblings."); + assertBuildFail("all(group(a) each(output(count() as(b),count() as(b))))", + "Can not use output label 'b' for multiple siblings."); + } + + @Test + public void requireThatOrderByReusesOutputResults() { + assertLayout("all(group(a) order(count()) each(output(count())))", + "[[{ Attribute, result = [Count], order = [[1], [AggregationRef]] }]]"); + assertLayout("all(group(a) order(count()) each(output(count() as(b))))", + "[[{ Attribute, result = [Count], order = [[1], [AggregationRef]] }]]"); + } + + @Test + public void requireThatNoopBranchesArePruned() { + assertLayout("all()", "[]"); + assertLayout("all(group(a))", "[]"); + assertLayout("all(group(a) each())", "[]"); + + String expectedA = "[{ Attribute, result = [Count] }]"; + assertLayout("all(group(a) each(output(count())))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all()))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b))))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())) as(foo)" + + " each())", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())) as(foo)" + + " each(group(b)))", + Arrays.asList(expectedA).toString()); + assertLayout("all(group(a) each(output(count()) all(group(b) each())) as(foo)" + + " each(group(b) each()))", + Arrays.asList(expectedA).toString()); + + String expectedB = "[{ Attribute }, { Attribute, result = [Count] }]"; + assertLayout("all(group(a) each(output(count()) all(group(b) each())) as(foo)" + + " each(group(b) each(output(count()))))", + Arrays.asList(expectedB, expectedA).toString()); + } + + @Test + public void requireThatAggregationLevelIsValidatedFails() { + assertBuildFail("all(group(artist) output(sum(length)))", + "Expression 'length' not applicable for single group."); + assertBuild("all(group(artist) each(output(count())))"); + assertBuildFail("all(group(artist) each(group(album) output(sum(length))))", + "Expression 'length' not applicable for single group."); + assertBuild("all(group(artist) each(group(album) each(output(count()))))"); + } + + @Test + public void requireThatCountOnListOfGroupsIsValidated() { + assertBuild("all(group(artist) output(count()))"); + assertBuild("all(group(artist) each(group(album) output(count())))"); + } + + @Test + public void requireThatGroupByIsValidated() { + assertBuild("all(group(artist) each(output(count())))"); + assertBuildFail("all(group(sum(artist)) each(output(count())))", + "Expression 'sum(artist)' not applicable for single hit."); + assertBuild("all(group(artist) each(group(album) each(output(count()))))"); + assertBuildFail("all(group(artist) each(group(sum(album)) each(output(count()))))", + "Expression 'sum(album)' not applicable for single hit."); + } + + @Test + public void requireThatGroupingLevelIsValidated() { + assertBuild("all(group(artist))"); + assertBuild("all(group(artist) each(group(album)))"); + assertBuildFail("all(group(artist) all(group(sum(price))))", + "Can not operate on list of list of groups."); + assertBuild("all(group(artist) each(group(album) each(group(song))))"); + assertBuildFail("all(group(artist) each(group(album) all(group(sum(price)))))", + "Can not operate on list of list of groups."); + } + + @Test + public void requireThatOrderByIsValidated() { + assertBuildFail("all(order(length))", + "Can not order single group content."); + assertBuild("all(group(artist) order(sum(length)))"); + assertBuildFail("all(group(artist) each(order(length)))", + "Can not order single group content."); + assertBuild("all(group(artist) each(group(album) order(sum(length))))"); + assertBuildFail("all(group(artist) each(group(album) each(order(length))))", + "Can not order single group content."); + } + + @Test + public void requireThatOrderByHasCorrectReference() { + assertOrderBy("all(group(a) order(count()) each(output(count())))", "[[[1]]]"); + assertOrderBy("all(group(a) order(-count()) each(output(count())))", "[[[-1]]]"); + assertOrderBy("all(group(a) order(count()) each(output(count(),sum(b))))", "[[[1]]]"); + assertOrderBy("all(group(a) order(-count()) each(output(count(),sum(b))))", "[[[-1]]]"); + assertOrderBy("all(group(a) order(count()) each(output(sum(b), count())))", "[[[1]]]"); + assertOrderBy("all(group(a) order(-count()) each(output(sum(b), count())))", "[[[-1]]]"); + + assertOrderBy("all(group(a) order(count(),sum(b)) each(output(count(),sum(b))))", "[[[1, 2]]]"); + assertOrderBy("all(group(a) order(count(),-sum(b)) each(output(count(),sum(b))))", "[[[1, -2]]]"); + assertOrderBy("all(group(a) order(-count(),sum(b)) each(output(count(),sum(b))))", "[[[-1, 2]]]"); + assertOrderBy("all(group(a) order(-count(),-sum(b)) each(output(count(),sum(b))))", "[[[-1, -2]]]"); + + // because order() is resolved before output(), index follows order() statement + assertOrderBy("all(group(a) order(count(),sum(b)) each(output(sum(b), count())))", "[[[1, 2]]]"); + assertOrderBy("all(group(a) order(count(),-sum(b)) each(output(sum(b), count())))", "[[[1, -2]]]"); + assertOrderBy("all(group(a) order(-count(),sum(b)) each(output(sum(b), count())))", "[[[-1, 2]]]"); + assertOrderBy("all(group(a) order(-count(),-sum(b)) each(output(sum(b), count())))", "[[[-1, -2]]]"); + + assertOrderBy("all(group(a) order(count()) each(output(count())) as(foo)" + + " each(output(sum(b))) as(bar))", + "[[[1]], [[1]]]"); + } + + + @Test + public void requireThatWhereIsValidated() { + assertBuild("all(where(true))"); + assertBuild("all(where($query))"); + assertBuildFail("all(where(foo))", + "Operation 'where' does not support 'foo'."); + assertBuildFail("all(group(artist) where(true))", + "Can not apply 'where' to non-root group."); + } + + @Test + public void requireThatRootAggregationCanBeTransformed() { + RequestTest test = new RequestTest(); + test.expectedOutput = CountAggregationResult.class.getName(); + test.request = "all(output(count()))"; + test.outputWriter = (groupingList, transform) -> groupingList.get(0).getRoot().getAggregationResults().get(0).getClass().getName(); + assertOutput(test); + } + + @Test + public void requireThatExpressionsCanBeLabeled() { + assertLabel("all(group(a) each(output(count())))", + "[[{ label = 'a', results = [count()] }]]"); + assertLabel("all(group(a) each(output(count())) as(b))", + "[[{ label = 'b', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(output(count()))))", + "[[{ label = 'a', results = [] }, { label = 'b', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(group(c) each(output(count())))))", + "[[{ label = 'a', results = [] }, { label = 'b', results = [] }, { label = 'c', results = [count()] }]]"); + assertBuildFail("all(group(a) each(output(count())) each(output(count())))", + "Can not use group list label 'a' for multiple siblings."); + assertBuildFail("all(all(group(a) each(output(count())))" + + " all(group(a) each(output(count()))))", + "Can not use group list label 'a' for multiple siblings."); + assertLabel("all(group(a) each(output(count())) as(a1)" + + " each(output(count())) as(a2))", + "[[{ label = 'a1', results = [count()] }], [{ label = 'a2', results = [count()] }]]"); + assertLabel("all(group(a) each(all(group(b) each(output(count())))" + + " all(group(c) each(output(count())))))", + "[[{ label = 'a', results = [] }, { label = 'b', results = [count()] }], [{ label = 'a', results = [] }, { label = 'c', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(output(count()))) as(a1)" + + " each(group(b) each(output(count()))) as(a2))", + "[[{ label = 'a1', results = [] }, { label = 'b', results = [count()] }], [{ label = 'a2', results = [] }, { label = 'b', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(group(c) each(output(count())))) as(a1)" + + " each(group(b) each(group(e) each(output(count())))) as(a2))", + "[[{ label = 'a1', results = [] }, { label = 'b', results = [] }, { label = 'c', results = [count()] }]," + + " [{ label = 'a2', results = [] }, { label = 'b', results = [] }, { label = 'e', results = [count()] }]]"); + assertLabel("all(group(a) each(group(b) each(output(count())) as(b1)" + + " each(output(count())) as(b2)))", + "[[{ label = 'a', results = [] }, { label = 'b1', results = [count()] }]," + + " [{ label = 'a', results = [] }, { label = 'b2', results = [count()] }]]"); + + assertBuildFail("all(group(a) each(each(output(summary() as(foo)))))", + "Can not label expression 'summary()'."); + assertLabel("all(group(foo) each(each(output(summary()))))", + "[[{ label = 'foo', results = [hits] }]]"); + assertLabel("all(group(foo) each(each(output(summary())) as(bar)))", + "[[{ label = 'foo', results = [bar] }]]"); + assertLabel("all(group(foo) each(each(output(summary())) as(bar)) as(baz))", + "[[{ label = 'baz', results = [bar] }]]"); + assertLabel("all(group(foo) each(each(output(summary())) as(bar)" + + " each(output(summary())) as(baz)))", + "[[{ label = 'foo', results = [bar] }]," + + " [{ label = 'foo', results = [baz] }]]"); + assertLabel("all(group(foo) each(each(output(summary())))" + + " each(each(output(summary()))) as(bar))", + "[[{ label = 'bar', results = [hits] }]," + + " [{ label = 'foo', results = [hits] }]]"); + } + + @Test + public void requireThatOrderByResultsAreNotLabeled() { + assertLabel("all(group(a) each(output(min(b), max(b), avg(b))))", + "[[{ label = 'a', results = [min(b), max(b), avg(b)] }]]"); + assertLabel("all(group(a) order(min(b)) each(output(max(b), avg(b))))", + "[[{ label = 'a', results = [max(b), avg(b), null] }]]"); + assertLabel("all(group(a) order(min(b), max(b)) each(output(avg(b))))", + "[[{ label = 'a', results = [avg(b), null, null] }]]"); + } + + @Test + public void requireThatTimeZoneIsAppliedToTimeFunctions() { + for (String timePart : Arrays.asList("dayofmonth", "dayofweek", "dayofyear", "hourofday", + "minuteofhour", "monthofyear", "secondofminute", "year")) + { + String request = "all(output(avg(time." + timePart + "(foo))))"; + assertTimeZone(request, "GMT-2", -7200L); + assertTimeZone(request, "GMT-1", -3600L); + assertTimeZone(request, "GMT", null); + assertTimeZone(request, "GMT+1", 3600L); + assertTimeZone(request, "GMT+2", 7200L); + } + } + + @Test + public void requireThatTimeDateIsExpanded() { + RequestTest test = new RequestTest(); + test.expectedOutput = new StrCatFunctionNode() + .addArg(new ToStringFunctionNode(new TimeStampFunctionNode(new AttributeNode("foo"), + TimeStampFunctionNode.TimePart.Year, true))) + .addArg(new ConstantNode(new StringResultNode("-"))) + .addArg(new ToStringFunctionNode(new TimeStampFunctionNode(new AttributeNode("foo"), + TimeStampFunctionNode.TimePart.Month, true))) + .addArg(new ConstantNode(new StringResultNode("-"))) + .addArg(new ToStringFunctionNode(new TimeStampFunctionNode(new AttributeNode("foo"), + TimeStampFunctionNode.TimePart.MonthDay, true))) + .toString(); + test.request = "all(output(avg(time.date(foo))))"; + test.outputWriter = (groupingList, transform) -> groupingList.get(0).getRoot().getAggregationResults().get(0).getExpression().toString(); + assertOutput(test); + } + + @Test + public void requireThatNowIsResolvedToCurrentTime() { + RequestTest test = new RequestTest(); + test.expectedOutput = Boolean.toString(true); + test.request = "all(output(avg(now() - foo)))"; + test.outputWriter = new OutputWriter() { + long before = System.currentTimeMillis(); + + @Override + public String write(List<Grouping> groupingList, GroupingTransform transform) { + AddFunctionNode add = + (AddFunctionNode)groupingList.get(0).getRoot().getAggregationResults().get(0).getExpression(); + long nowValue = ((ConstantNode)add.getArg(0)).getValue().getInteger(); + boolean preCond = nowValue >= (before / 1000); + long after = System.currentTimeMillis(); + boolean postCond = nowValue <= (after / 1000); + boolean allOk = preCond && postCond; + return Boolean.toString(allOk); + } + }; + assertOutput(test); + } + + private static CompositeContinuation newComposite(EncodableContinuation... conts) { + CompositeContinuation ret = new CompositeContinuation(); + for (EncodableContinuation cont : conts) { + ret.add(cont); + } + return ret; + } + + private static OffsetContinuation newOffset(int tag, int offset) { + return new OffsetContinuation(ResultId.valueOf(0), tag, offset, 0); + } + + private static OffsetContinuation newUnstableOffset(int tag, int offset) { + return new OffsetContinuation(ResultId.valueOf(0), tag, offset, OffsetContinuation.FLAG_UNSTABLE); + } + + private static void assertBuild(String request) { + RequestTest test = new RequestTest(); + test.request = request; + assertOutput(test); + } + + private static void assertBuildFail(String request, String expectedException) { + RequestTest test = new RequestTest(); + test.request = request; + test.expectedException = expectedException; + assertOutput(test); + } + + private static void assertTimeZone(String request, String timeZone, Long expectedOutput) { + RequestTest test = new RequestTest(); + test.request = request; + test.timeZone = timeZone; + test.outputWriter = (groupingList, transform) -> { + Long timeOffset = null; + ExpressionNode node = + ((TimeStampFunctionNode)groupingList.get(0).getRoot().getAggregationResults().get(0) + .getExpression()).getArg(0); + if (node instanceof AddFunctionNode) { + timeOffset = (((ConstantNode)((AddFunctionNode)node).getArg(1)).getValue()).getInteger(); + } + return String.valueOf(timeOffset); + }; + test.expectedOutput = String.valueOf(expectedOutput); + assertOutput(test); + } + + private static void assertLabel(String request, String expectedOutput) { + assertOutput(request, new LabelWriter(), expectedOutput); + } + + private static void assertLayout(String request, String expectedOutput) { + assertOutput(request, new LayoutWriter(), expectedOutput); + } + + private static void assertOrderBy(String request, String expectedOutput) { + assertOutput(request, new OrderByWriter(), expectedOutput); + } + + private static void assertOffset(String request, Continuation continuation, String expectedOutput) { + RequestTest ret = new RequestTest(); + ret.request = request; + ret.continuation = continuation; + ret.outputWriter = new OffsetWriter(); + ret.expectedOutput = expectedOutput; + assertOutput(ret); + } + + private static void assertForceSinglePass(String request, String expectedOutput) { + assertOutput(request, new ForceSinglePassWriter(), expectedOutput); + } + + private static void assertOutput(String request, OutputWriter writer, String expectedOutput) { + RequestTest ret = new RequestTest(); + ret.request = request; + ret.outputWriter = writer; + ret.expectedOutput = expectedOutput; + assertOutput(ret); + } + + private static void assertOutput(RequestTest test) { + RequestBuilder builder = new RequestBuilder(0); + builder.setRootOperation(GroupingOperation.fromString(test.request)); + builder.setTimeZone(TimeZone.getTimeZone(test.timeZone)); + builder.addContinuations(Arrays.asList(test.continuation)); + try { + builder.build(); + if (test.expectedException != null) { + fail("Expected exception '" + test.expectedException + "'."); + } + } catch (RuntimeException e) { + if (test.expectedException == null) { + throw e; + } + assertEquals(test.expectedException, e.getMessage()); + return; + } + if (test.outputWriter != null) { + String output = test.outputWriter.write(builder.getRequestList(), builder.getTransform()); + assertEquals(test.expectedOutput, output); + } + } + + private static class RequestTest { + + String request; + String timeZone = "utc"; + String expectedException; + String expectedOutput; + OutputWriter outputWriter; + Continuation continuation; + } + + private static interface OutputWriter { + + String write(List<Grouping> groupingList, GroupingTransform transform); + } + + private static class OffsetWriter implements OutputWriter { + + @Override + public String write(List<Grouping> groupingList, GroupingTransform transform) { + List<String> foo = new LinkedList<>(); + for (Grouping grouping : groupingList) { + List<String> bar = new LinkedList<>(); + for (GroupingLevel level : grouping.getLevels()) { + List<String> baz = new LinkedList<>(); + for (AggregationResult result : level.getGroupPrototype().getAggregationResults()) { + if (result instanceof HitsAggregationResult) { + int tag = result.getTag(); + baz.add("{ tag = " + tag + ", max = [" + transform.getMax(tag) + ", " + + ((HitsAggregationResult)result).getMaxHits() + "] }"); + } + } + int tag = level.getGroupPrototype().getTag(); + bar.add("{ tag = " + tag + ", max = [" + transform.getMax(tag) + ", " + level.getMaxGroups() + + "], hits = " + baz.toString() + " }"); + } + foo.add(bar.toString()); + } + Collections.sort(foo); + return foo.toString(); + } + } + + private static class LabelWriter implements OutputWriter { + + @Override + public String write(List<Grouping> groupingList, GroupingTransform transform) { + List<String> foo = new LinkedList<>(); + for (Grouping grouping : groupingList) { + List<String> bar = new LinkedList<>(); + for (GroupingLevel level : grouping.getLevels()) { + List<String> baz = new LinkedList<>(); + for (AggregationResult result : level.getGroupPrototype().getAggregationResults()) { + baz.add(transform.getLabel(result.getTag())); + } + bar.add("{ label = '" + transform.getLabel(level.getGroupPrototype().getTag()) + + "', results = " + baz.toString() + " }"); + } + foo.add(bar.toString()); + } + Collections.sort(foo); + return foo.toString(); + } + } + + private static class LayoutWriter implements OutputWriter { + + @Override + public String write(List<Grouping> groupingList, GroupingTransform transform) { + List<String> foo = new LinkedList<>(); + for (Grouping grouping : groupingList) { + List<String> bar = new LinkedList<>(); + for (GroupingLevel level : grouping.getLevels()) { + StringBuilder str = new StringBuilder("{ "); + str.append(toSimpleName(level.getExpression())).append(", "); + if (level.getMaxGroups() >= 0 || level.getPrecision() >= 0) { + str.append("max = [").append(level.getMaxGroups()).append(", ") + .append(level.getPrecision()).append("], "); + } + Group group = level.getGroupPrototype(); + if (!group.getAggregationResults().isEmpty()) { + List<String> baz = new LinkedList<>(); + for (AggregationResult exp : level.getGroupPrototype().getAggregationResults()) { + baz.add(toSimpleName(exp)); + } + str.append("result = ").append(baz).append(", "); + } + if (!group.getOrderByIndexes().isEmpty() || !group.getOrderByExpressions().isEmpty()) { + List<String> baz = new LinkedList<>(); + for (Integer idx : level.getGroupPrototype().getOrderByIndexes()) { + baz.add(idx.toString()); + } + str.append("order = [").append(baz).append(", "); + baz = new LinkedList<>(); + for (ExpressionNode exp : level.getGroupPrototype().getOrderByExpressions()) { + baz.add(toSimpleName(exp)); + } + str.append(baz).append("], "); + } + str.setLength(str.length() - 2); + str.append(" }"); + bar.add(str.toString()); + } + foo.add(bar.toString()); + } + Collections.sort(foo); + return foo.toString(); + } + + private static String toSimpleName(ExpressionNode exp) { + String ret = exp.getClass().getSimpleName(); + if (ret.endsWith("AggregationResult")) { + return ret.substring(0, ret.length() - 17); + } + if (ret.endsWith("FunctionNode")) { + return ret.substring(0, ret.length() - 12); + } + if (ret.endsWith("Node")) { + return ret.substring(0, ret.length() - 4); + } + return ret; + } + } + + private static class OrderByWriter implements OutputWriter { + + @Override + public String write(List<Grouping> groupingList, GroupingTransform transform) { + List<List<String>> ret = new LinkedList<>(); + for (Grouping grouping : groupingList) { + List<String> lst = new LinkedList<>(); + for (GroupingLevel level : grouping.getLevels()) { + lst.add(level.getGroupPrototype().getOrderByIndexes().toString()); + } + ret.add(lst); + } + return ret.toString(); + } + } + + private static class ForceSinglePassWriter implements OutputWriter { + + @Override + public String write(List<Grouping> groupingList, GroupingTransform transform) { + List<String> ret = new LinkedList<>(); + for (Grouping grouping : groupingList) { + ret.add(String.valueOf(grouping.getForceSinglePass())); + } + return ret.toString(); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java new file mode 100644 index 00000000000..d8438ddc042 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java @@ -0,0 +1,1108 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import com.yahoo.document.GlobalId; +import com.yahoo.document.idstring.IdString; +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.grouping.result.AbstractList; +import com.yahoo.search.grouping.result.GroupList; +import com.yahoo.search.grouping.result.HitList; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.result.Relevance; +import com.yahoo.searchlib.aggregation.*; +import com.yahoo.searchlib.aggregation.hll.SparseSketch; +import com.yahoo.searchlib.expression.*; +import org.junit.Test; + +import java.util.*; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ResultBuilderTestCase { + + private static final int REQUEST_ID = 0; + private static final int ROOT_IDX = 0; + + @Test + public void requireThatAllGroupIdsCanBeConverted() { + assertGroupId("group:6.9", new FloatResultNode(6.9)); + assertGroupId("group:69", new IntegerResultNode(69)); + assertGroupId("group:null", new NullResultNode()); + assertGroupId("group:[6, 9]", new RawResultNode(new byte[] { 6, 9 })); + assertGroupId("group:a", new StringResultNode("a")); + assertGroupId("group:6.9:9.6", new FloatBucketResultNode(6.9, 9.6)); + assertGroupId("group:6:9", new IntegerBucketResultNode(6, 9)); + assertGroupId("group:a:b", new StringBucketResultNode("a", "b")); + assertGroupId("group:[6, 9]:[9, 6]", new RawBucketResultNode(new RawResultNode(new byte[] { 6, 9 }), + new RawResultNode(new byte[] { 9, 6 }))); + } + + @Test + public void requireThatUnknownGroupIdThrows() { + assertBuildFail("all(group(a) each(output(count())))", + Arrays.asList(newGrouping(new Group().setTag(2).setId(new MyResultNode()))), + "com.yahoo.search.grouping.vespa.ResultBuilderTestCase$MyResultNode"); + } + + @Test + public void requireThatAllExpressionNodesCanBeConverted() { + assertResult("0", new AverageAggregationResult(new IntegerResultNode(6), 9)); + assertResult("69", new CountAggregationResult(69)); + assertResult("69", new MaxAggregationResult(new IntegerResultNode(69))); + assertResult("69", new MinAggregationResult(new IntegerResultNode(69))); + assertResult("69", new SumAggregationResult(new IntegerResultNode(69))); + assertResult("69", new XorAggregationResult(69)); + assertResult("69", new ExpressionCountAggregationResult(new SparseSketch(), sketch -> 69)); + } + + @Test + public void requireThatUnknownExpressionNodeThrows() { + assertBuildFail("all(group(a) each(output(count())))", + Arrays.asList(newGrouping(newGroup(2, 2, new MyAggregationResult().setTag(3)))), + "com.yahoo.search.grouping.vespa.ResultBuilderTestCase$MyAggregationResult"); + } + + @Test + public void requireThatRootResultsAreIncluded() { + assertLayout("all(output(count()))", + new Grouping().setRoot(newGroup(1, new CountAggregationResult(69).setTag(2))), + "RootGroup{id=group:root, count()=69}[]"); + } + + @Test + public void requireThatRootResultsAreIncludedUsingExpressionCountAggregationResult() { + assertLayout("all(group(a) output(count()))", + new Grouping().setRoot(newGroup(1, new ExpressionCountAggregationResult(new SparseSketch(), sketch -> 69).setTag(2))), + "RootGroup{id=group:root, count()=69}[]"); + } + + @Test + public void requireThatNestedGroupingResultsCanBeTransformed() { + Grouping grouping = new Grouping() + .setRoot(new Group() + .setTag(1) + .addChild(new Group() + .setTag(2) + .setId(new StringResultNode("foo")) + .addAggregationResult(new CountAggregationResult(10).setTag(3)) + .addChild(new Group() + .setTag(4) + .setId(new StringResultNode("foo_a")) + .addAggregationResult(new CountAggregationResult(15) + .setTag(5))) + .addChild(new Group() + .setTag(4) + .setId(new StringResultNode("foo_b")) + .addAggregationResult(new CountAggregationResult(16) + .setTag(5)))) + .addChild(new Group() + .setTag(2) + .setId(new StringResultNode("bar")) + .addAggregationResult(new CountAggregationResult(20).setTag(3)) + .addChild(new Group() + .setTag(4) + .setId(new StringResultNode("bar_a")) + .addAggregationResult( + new CountAggregationResult(25) + .setTag(5))) + .addChild(new Group() + .setTag(4) + .setId(new StringResultNode("bar_b")) + .addAggregationResult( + new CountAggregationResult(26) + .setTag(5))))); + assertLayout("all(group(artist) max(5) each(output(count() as(baz)) all(group(album) " + + "max(5) each(output(count() as(cox))) as(group_album))) as(group_artist))", + grouping, + "RootGroup{id=group:root}[GroupList{label=group_artist}[" + + "Group{id=group:foo, baz=10}[GroupList{label=group_album}[Group{id=group:foo_a, cox=15}[], Group{id=group:foo_b, cox=16}[]]], " + + "Group{id=group:bar, baz=20}[GroupList{label=group_album}[Group{id=group:bar_a, cox=25}[], Group{id=group:bar_b, cox=26}[]]]]]"); + } + + @Test + public void requireThatParallelResultsAreTransformed() { + assertBuild("all(group(foo) each(output(count())) as(bar) each(output(count())) as(baz))", + Arrays.asList(new Grouping().setRoot(newGroup(1, 0)), + new Grouping().setRoot(newGroup(1, 0)))); + assertBuildFail("all(group(foo) each(output(count())) as(bar) each(output(count())) as(baz))", + Arrays.asList(new Grouping().setRoot(newGroup(2)), + new Grouping().setRoot(newGroup(3))), + "Expected 1 group, got 2."); + } + + @Test + public void requireThatTagsAreHandledCorrectly() { + assertBuild("all(group(a) each(output(count())))", + Arrays.asList(newGrouping( + newGroup(7, new CountAggregationResult(0))))); + } + + @Test + public void requireThatEmptyBranchesArePruned() { + assertBuildFail("all()", Collections.<Grouping>emptyList(), "Expected 1 group, got 0."); + assertBuildFail("all(group(a))", Collections.<Grouping>emptyList(), "Expected 1 group, got 0."); + assertBuildFail("all(group(a) each())", Collections.<Grouping>emptyList(), "Expected 1 group, got 0."); + + Grouping grouping = newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))); + String expectedOutput = "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=69}[]]]"; + assertLayout("all(group(a) each(output(count())))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all()))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b))))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each())))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each())))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each()))" + + " each() as(foo))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each()))" + + " each(group(b)) as(foo))", grouping, expectedOutput); + assertLayout("all(group(a) each(output(count()) all(group(b) each()))" + + " each(group(b) each()) as(foo))", grouping, expectedOutput); + } + + @Test + public void requireThatGroupListsAreLabeled() { + assertLayout("all(group(a) each(output(count())))", + newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=69}[]]]"); + assertLayout("all(group(a) each(output(count())) as(bar))", + newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))), + "RootGroup{id=group:root}[GroupList{label=bar}[Group{id=group:2, count()=69}[]]]"); + } + + @Test + public void requireThatHitListsAreLabeled() { + assertLayout("all(group(foo) each(each(output(summary()))))", + newGrouping(newGroup(2, newHitList(3, 2))), + "RootGroup{id=group:root}[GroupList{label=foo}[Group{id=group:2}[" + + "HitList{label=hits}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + assertLayout("all(group(foo) each(each(output(summary())) as(bar)))", + newGrouping(newGroup(2, newHitList(3, 2))), + "RootGroup{id=group:root}[GroupList{label=foo}[Group{id=group:2}[" + + "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + assertLayout("all(group(foo) each(each(output(summary())) as(bar)) as(baz))", + newGrouping(newGroup(2, newHitList(3, 2))), + "RootGroup{id=group:root}[GroupList{label=baz}[Group{id=group:2}[" + + "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + assertLayout("all(group(foo) each(each(output(summary())) as(bar)" + + " each(output(summary())) as(baz)))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 2))), + newGrouping(newGroup(2, newHitList(4, 2)))), + "RootGroup{id=group:root}[GroupList{label=foo}[Group{id=group:2}[" + + "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}], " + + "HitList{label=baz}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + assertLayout("all(group(foo) each(each(output(summary())))" + + " each(each(output(summary()))) as(bar))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 2))), + newGrouping(newGroup(4, newHitList(5, 2)))), + "RootGroup{id=group:root}[" + + "GroupList{label=foo}[Group{id=group:2}[HitList{label=hits}[Hit{id=hit:1}, Hit{id=hit:2}]]], " + + "GroupList{label=bar}[Group{id=group:4}[HitList{label=hits}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); + } + + @Test + public void requireThatOutputsAreLabeled() { + assertLayout("all(output(count()))", + new Grouping().setRoot(newGroup(1, new CountAggregationResult(69).setTag(2))), + "RootGroup{id=group:root, count()=69}[]"); + assertLayout("all(output(count() as(foo)))", + new Grouping().setRoot(newGroup(1, new CountAggregationResult(69).setTag(2))), + "RootGroup{id=group:root, foo=69}[]"); + assertLayout("all(group(a) each(output(count())))", + newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=69}[]]]"); + assertLayout("all(group(a) each(output(count() as(foo))))", + newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, foo=69}[]]]"); + } + + @Test + public void requireThatExpressionCountCanUseExactGroupCount() { + Group root1 = newGroup(1, new ExpressionCountAggregationResult(new SparseSketch(), sketch -> 42).setTag(2)); + Grouping grouping1 = new Grouping().setRoot(root1); + + // Should return estimate when no groups are returned (since each() clause is absent). + assertLayout("all(group(artist) output(count()))", + grouping1, + "RootGroup{id=group:root, count()=42}[]"); + + Group root2 = newGroup(1, new ExpressionCountAggregationResult(new SparseSketch(), sketch -> 42).setTag(2)); + Grouping grouping2 = new Grouping().setRoot(root2); + for (int i = 0; i < 3; ++i) { + root2.addChild(new Group() + .setTag(2) + .setId(new StringResultNode("foo" + i))) + .addAggregationResult(new CountAggregationResult(i).setTag(3)); + } + + // Should return the number of groups when max is not present. + assertLayout("all(group(artist) output(count()) each(output(count())))", + grouping2, + "RootGroup{id=group:root, count()=3, artist=2}" + + "[GroupList{label=count()}[Group{id=group:foo0}[], Group{id=group:foo1}[], Group{id=group:foo2}[]]]"); + + // Should return the number of groups when max is higher than group count. + assertLayout("all(group(artist) max(5) output(count()) each(output(count())))", + grouping2, + "RootGroup{id=group:root, count()=3, artist=2}" + + "[GroupList{label=count()}[Group{id=group:foo0}[], Group{id=group:foo1}[], Group{id=group:foo2}[]]]"); + + // Should return the estimate when number of groups is equal to max. + assertLayout("all(group(artist) max(3) output(count()) each(output(count())))", + grouping2, + "RootGroup{id=group:root, count()=42, artist=2}" + + "[GroupList{label=count()}[Group{id=group:foo0}[], Group{id=group:foo1}[], Group{id=group:foo2}[]]]"); + + } + + + @Test + public void requireThatResultContinuationContainsCurrentPages() { + String request = "all(group(a) max(2) each(output(count())))"; + Grouping result = newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + newGroup(2, 2, new CountAggregationResult(2)), + newGroup(2, 3, new CountAggregationResult(3)), + newGroup(2, 4, new CountAggregationResult(4))); + assertResultCont(request, result, newOffset(newResultId(0), 2, 0), "[]"); + assertResultCont(request, result, newOffset(newResultId(0), 2, 1), "[0=1]"); + assertResultCont(request, result, newOffset(newResultId(0), 2, 2), "[0=2]"); + assertResultCont(request, result, newOffset(newResultId(0), 2, 3), "[0=3]"); + + assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), + "[]"); + assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), + newOffset(newResultId(0), 2, 1), + "[0=1]"); + assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + + " each(output(count())) as(bar))", + Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), + newComposite(newOffset(newResultId(0), 2, 2), + newOffset(newResultId(1), 4, 1)), + "[0=2, 1=1]"); + + request = "all(group(a) each(max(2) each(output(summary()))))"; + result = newGrouping(newGroup(2, newHitList(3, 4))); + assertResultCont(request, result, newOffset(newResultId(0, 0, 0), 3, 0), "[]"); + assertResultCont(request, result, newOffset(newResultId(0, 0, 0), 3, 1), "[0.0.0=1]"); + assertResultCont(request, result, newOffset(newResultId(0, 0, 0), 3, 2), "[0.0.0=2]"); + assertResultCont(request, result, newOffset(newResultId(0, 0, 0), 3, 3), "[0.0.0=3]"); + + assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + + " each(max(2) each(output(summary()))) as(bar))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + newGrouping(newGroup(4, newHitList(5, 4)))), + "[]"); + assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + + " each(max(2) each(output(summary()))) as(bar))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + newGrouping(newGroup(4, newHitList(5, 4)))), + newOffset(newResultId(0, 0, 0), 3, 1), + "[0.0.0=1]"); + assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + + " each(max(2) each(output(summary()))) as(bar))", + Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + newGrouping(newGroup(4, newHitList(5, 4)))), + newComposite(newOffset(newResultId(0, 0, 0), 3, 2), + newOffset(newResultId(1, 0, 0), 5, 1)), + "[0.0.0=2, 1.0.0=1]"); + } + + @Test + public void requireThatGroupListContinuationsAreNotCreatedWhenUnlessMaxIsSet() { + assertContinuation("all(group(a) each(output(count())))", + newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + newGroup(2, 2, new CountAggregationResult(2)), + newGroup(2, 3, new CountAggregationResult(3)), + newGroup(2, 4, new CountAggregationResult(4))), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:1', {}, [] }, { 'group:2', {}, [] }, { 'group:3', {}, [] }, { 'group:4', {}, [] }] }] }"); + } + + @Test + public void requireThatGroupListContinuationsCanBeSet() { + String request = "all(group(a) max(2) each(output(count())))"; + Grouping result = newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + newGroup(2, 2, new CountAggregationResult(2)), + newGroup(2, 3, new CountAggregationResult(3)), + newGroup(2, 4, new CountAggregationResult(4))); + assertContinuation(request, result, newOffset(newResultId(0), 2, 0), + "{ 'group:root', {}, [{ 'grouplist:a', {next=2}, [" + + "{ 'group:1', {}, [] }, { 'group:2', {}, [] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 1), + "{ 'group:root', {}, [{ 'grouplist:a', {next=3, prev=0}, [" + + "{ 'group:2', {}, [] }, { 'group:3', {}, [] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 2), + "{ 'group:root', {}, [{ 'grouplist:a', {prev=0}, [" + + "{ 'group:3', {}, [] }, { 'group:4', {}, [] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 3), + "{ 'group:root', {}, [{ 'grouplist:a', {prev=1}, [" + + "{ 'group:4', {}, [] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 4), + "{ 'group:root', {}, [{ 'grouplist:a', {prev=2}, [" + + "] }] }"); + assertContinuation(request, result, newOffset(newResultId(0), 2, 5), + "{ 'group:root', {}, [{ 'grouplist:a', {prev=2}, [" + + "] }] }"); + } + + @Test + public void requireThatGroupListContinuationsCanBeSetInSiblingGroups() { + String request = "all(group(a) each(group(b) max(2) each(output(count()))))"; + Grouping result = newGrouping(newGroup(2, 201, + newGroup(3, 301, new CountAggregationResult(1)), + newGroup(3, 302, new CountAggregationResult(2)), + newGroup(3, 303, new CountAggregationResult(3)), + newGroup(3, 304, new CountAggregationResult(4))), + newGroup(2, 202, + newGroup(3, 305, new CountAggregationResult(5)), + newGroup(3, 306, new CountAggregationResult(6)), + newGroup(3, 307, new CountAggregationResult(7)), + newGroup(3, 308, new CountAggregationResult(8)))); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 0), + newOffset(newResultId(0, 1, 0), 2, 5)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [{ 'group:301', {}, [] }, { 'group:302', {}, [] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 1), + newOffset(newResultId(0, 1, 0), 2, 4)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=3, prev=0}, [{ 'group:302', {}, [] }, { 'group:303', {}, [] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 2), + newOffset(newResultId(0, 1, 0), 2, 3)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [{ 'group:303', {}, [] }, { 'group:304', {}, [] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {prev=1}, [{ 'group:308', {}, [] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 3), + newOffset(newResultId(0, 1, 0), 2, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=1}, [{ 'group:304', {}, [] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {prev=0}, [{ 'group:307', {}, [] }, { 'group:308', {}, [] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 4), + newOffset(newResultId(0, 1, 0), 2, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=2}, [] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=3, prev=0}, [{ 'group:306', {}, [] }, { 'group:307', {}, [] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 2, 5), + newOffset(newResultId(0, 1, 0), 2, 0)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=2}, [] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [{ 'group:305', {}, [] }, { 'group:306', {}, [] }] }] }] }] }"); + } + + @Test + public void requireThatGroupListContinuationsCanBeSetInSiblingGroupLists() { + String request = "all(group(a) max(2) each(output(count())) as(foo)" + + " each(output(count())) as(bar))"; + List<Grouping> result = Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + newGroup(2, 2, new CountAggregationResult(2)), + newGroup(2, 3, new CountAggregationResult(3)), + newGroup(2, 4, new CountAggregationResult(4))), + newGrouping(newGroup(4, 1, new CountAggregationResult(1)), + newGroup(4, 2, new CountAggregationResult(2)), + newGroup(4, 3, new CountAggregationResult(3)), + newGroup(4, 4, new CountAggregationResult(4)))); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 0), + newOffset(newResultId(1), 4, 5)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {next=2}, [{ 'group:1', {}, [] }, { 'group:2', {}, [] }] }, " + + "{ 'grouplist:bar', {prev=2}, [] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 1), + newOffset(newResultId(1), 4, 4)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {next=3, prev=0}, [{ 'group:2', {}, [] }, { 'group:3', {}, [] }] }, " + + "{ 'grouplist:bar', {prev=2}, [] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 2), + newOffset(newResultId(1), 4, 3)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {prev=0}, [{ 'group:3', {}, [] }, { 'group:4', {}, [] }] }, " + + "{ 'grouplist:bar', {prev=1}, [{ 'group:4', {}, [] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 3), + newOffset(newResultId(1), 4, 2)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {prev=1}, [{ 'group:4', {}, [] }] }, " + + "{ 'grouplist:bar', {prev=0}, [{ 'group:3', {}, [] }, { 'group:4', {}, [] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 4), + newOffset(newResultId(1), 4, 1)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {prev=2}, [] }, " + + "{ 'grouplist:bar', {next=3, prev=0}, [{ 'group:2', {}, [] }, { 'group:3', {}, [] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0), 2, 5), + newOffset(newResultId(1), 4, 0)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {prev=2}, [] }, " + + "{ 'grouplist:bar', {next=2}, [{ 'group:1', {}, [] }, { 'group:2', {}, [] }] }] }"); + } + + @Test + public void requireThatUnstableContinuationsDoNotAffectSiblingGroupLists() { + String request = "all(group(a) each(group(b) max(2) each(group(c) max(2) each(output(count())))))"; + Grouping result = newGrouping(newGroup(2, 201, + newGroup(3, 301, + newGroup(4, 401, new CountAggregationResult(1)), + newGroup(4, 402, new CountAggregationResult(1)), + newGroup(4, 403, new CountAggregationResult(1)), + newGroup(4, 404, new CountAggregationResult(1))), + newGroup(3, 302, + newGroup(4, 405, new CountAggregationResult(1)), + newGroup(4, 406, new CountAggregationResult(1)), + newGroup(4, 407, new CountAggregationResult(1)), + newGroup(4, 408, new CountAggregationResult(1))), + newGroup(3, 303, + newGroup(4, 409, new CountAggregationResult(1)), + newGroup(4, 410, new CountAggregationResult(1)), + newGroup(4, 411, new CountAggregationResult(1)), + newGroup(4, 412, new CountAggregationResult(1))), + newGroup(3, 304, + newGroup(4, 413, new CountAggregationResult(1)), + newGroup(4, 414, new CountAggregationResult(1)), + newGroup(4, 415, new CountAggregationResult(1)), + newGroup(4, 416, new CountAggregationResult(1)))), + newGroup(2, 202, + newGroup(3, 305, + newGroup(4, 417, new CountAggregationResult(1)), + newGroup(4, 418, new CountAggregationResult(1)), + newGroup(4, 419, new CountAggregationResult(1)), + newGroup(4, 420, new CountAggregationResult(1))), + newGroup(3, 306, + newGroup(4, 421, new CountAggregationResult(1)), + newGroup(4, 422, new CountAggregationResult(1)), + newGroup(4, 423, new CountAggregationResult(1)), + newGroup(4, 424, new CountAggregationResult(1))), + newGroup(3, 307, + newGroup(4, 425, new CountAggregationResult(1)), + newGroup(4, 426, new CountAggregationResult(1)), + newGroup(4, 427, new CountAggregationResult(1)), + newGroup(4, 428, new CountAggregationResult(1))), + newGroup(3, 308, + newGroup(4, 429, new CountAggregationResult(1)), + newGroup(4, 430, new CountAggregationResult(1)), + newGroup(4, 431, new CountAggregationResult(1)), + newGroup(4, 432, new CountAggregationResult(1))))); + assertContinuation(request, result, newComposite(), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {next=2}, [{ 'group:401', {}, [] }, { 'group:402', {}, [] }] }] }, " + + "{ 'group:302', {}, [{ 'grouplist:c', {next=2}, [{ 'group:405', {}, [] }, { 'group:406', {}, [] }] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'grouplist:c', {next=2}, [{ 'group:417', {}, [] }, { 'group:418', {}, [] }] }] }, " + + "{ 'group:306', {}, [{ 'grouplist:c', {next=2}, [{ 'group:421', {}, [] }, { 'group:422', {}, [] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {next=2}, [{ 'group:401', {}, [] }, { 'group:402', {}, [] }] }] }, " + + "{ 'group:302', {}, [{ 'grouplist:c', {next=2}, [{ 'group:405', {}, [] }, { 'group:406', {}, [] }] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'grouplist:c', {next=2}, [{ 'group:417', {}, [] }, { 'group:418', {}, [] }] }] }, " + + "{ 'group:306', {}, [{ 'grouplist:c', {prev=0}, [{ 'group:423', {}, [] }, { 'group:424', {}, [] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2), + newOffset(newResultId(0, 0, 0), 2, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:303', {}, [{ 'grouplist:c', {next=2}, [{ 'group:409', {}, [] }, { 'group:410', {}, [] }] }] }, " + + "{ 'group:304', {}, [{ 'grouplist:c', {next=2}, [{ 'group:413', {}, [] }, { 'group:414', {}, [] }] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'grouplist:c', {next=2}, [{ 'group:417', {}, [] }, { 'group:418', {}, [] }] }] }, " + + "{ 'group:306', {}, [{ 'grouplist:c', {prev=0}, [{ 'group:423', {}, [] }, { 'group:424', {}, [] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2), + newOffset(newResultId(0, 0, 0), 2, 2), + newUnstableOffset(newResultId(0, 1, 0), 2, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:303', {}, [{ 'grouplist:c', {next=2}, [{ 'group:409', {}, [] }, { 'group:410', {}, [] }] }] }, " + + "{ 'group:304', {}, [{ 'grouplist:c', {next=2}, [{ 'group:413', {}, [] }, { 'group:414', {}, [] }] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=3, prev=0}, [" + + "{ 'group:306', {}, [{ 'grouplist:c', {next=2}, [{ 'group:421', {}, [] }, { 'group:422', {}, [] }] }] }, " + + "{ 'group:307', {}, [{ 'grouplist:c', {next=2}, [{ 'group:425', {}, [] }, { 'group:426', {}, [] }] }] }] }] }] }] }"); + } + + @Test + public void requireThatUnstableContinuationsAffectAllDecendants() { + String request = "all(group(a) each(group(b) max(1) each(group(c) max(1) each(group(d) max(1) each(output(count()))))))"; + Grouping result = newGrouping(newGroup(2, 201, + newGroup(3, 301, + newGroup(4, 401, + newGroup(5, 501, new CountAggregationResult(1)), + newGroup(5, 502, new CountAggregationResult(1))), + newGroup(4, 402, + newGroup(5, 503, new CountAggregationResult(1)), + newGroup(5, 504, new CountAggregationResult(1)))), + newGroup(3, 302, + newGroup(4, 403, + newGroup(5, 505, new CountAggregationResult(1)), + newGroup(5, 506, new CountAggregationResult(1))), + newGroup(4, 404, + newGroup(5, 507, new CountAggregationResult(1)), + newGroup(5, 508, new CountAggregationResult(1)))))); + assertContinuation(request, result, newComposite(), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=1}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {next=1}, [" + + "{ 'group:401', {}, [{ 'grouplist:d', {next=1}, [" + + "{ 'group:501', {}, [] }] }] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0, 0, 0, 0, 0), 5, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=1}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {next=1}, [" + + "{ 'group:401', {}, [{ 'grouplist:d', {prev=0}, [" + + "{ 'group:502', {}, [] }] }] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0, 0, 0, 0, 0), 5, 1), + newUnstableOffset(newResultId(0, 0, 0, 0, 0), 4, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=1}, [" + + "{ 'group:301', {}, [{ 'grouplist:c', {prev=0}, [" + + "{ 'group:402', {}, [{ 'grouplist:d', {next=1}, [" + + "{ 'group:503', {}, [] }] }] }] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0, 0, 0, 0, 0), 5, 1), + newUnstableOffset(newResultId(0, 0, 0, 0, 0), 4, 1), + newUnstableOffset(newResultId(0, 0, 0), 3, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:302', {}, [{ 'grouplist:c', {next=1}, [" + + "{ 'group:403', {}, [{ 'grouplist:d', {next=1}, [" + + "{ 'group:505', {}, [] }] }] }] }] }] }] }] }] }"); + } + + @Test + public void requireThatHitListContinuationsAreNotCreatedUnlessMaxIsSet() { + assertContinuation("all(group(a) each(each(output(summary()))))", + newGrouping(newGroup(2, newHitList(3, 4))), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {}, [hit:1, hit:2, hit:3, hit:4] }] }] }] }"); + } + + @Test + public void requireThatHitListContinuationsCanBeSet() { + String request = "all(group(a) each(max(2) each(output(summary()))))"; + Grouping result = newGrouping(newGroup(2, newHitList(3, 4))); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 0), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 1), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 2), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 3), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 4), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newOffset(newResultId(0, 0, 0), 3, 5), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:2', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + } + + @Test + public void requireThatHitListContinuationsCanBeSetInSiblingGroups() { + String request = "all(group(a) each(max(2) each(output(summary()))))"; + Grouping result = newGrouping(newGroup(2, 201, newHitList(3, 4)), + newGroup(2, 202, newHitList(3, 4))); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 0), + newOffset(newResultId(0, 1, 0), 3, 5)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 1), + newOffset(newResultId(0, 1, 0), 3, 4)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 2), + newOffset(newResultId(0, 1, 0), 3, 3)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 3), + newOffset(newResultId(0, 1, 0), 3, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 4), + newOffset(newResultId(0, 1, 0), 3, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {prev=2}, [] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 5), + newOffset(newResultId(0, 1, 0), 3, 0)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'hitlist:hits', {prev=2}, [] }] }, " + + "{ 'group:202', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }"); + } + + @Test + public void requireThatHitListContinuationsCanBeSetInSiblingHitLists() { + String request = "all(group(a) each(max(2) each(output(summary()))) as(foo)" + + " each(max(2) each(output(summary()))) as(bar))"; + List<Grouping> result = Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + newGrouping(newGroup(4, newHitList(5, 4)))); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 0), + newOffset(newResultId(1, 0, 0), 5, 5)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 1), + newOffset(newResultId(1, 0, 0), 5, 4)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 2), + newOffset(newResultId(1, 0, 0), 5, 3)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 3), + newOffset(newResultId(1, 0, 0), 5, 2)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {prev=1}, [hit:4] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 4), + newOffset(newResultId(1, 0, 0), 5, 1)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {next=3, prev=0}, [hit:2, hit:3] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 5), + newOffset(newResultId(1, 0, 0), 5, 0)), + "{ 'group:root', {}, [" + + "{ 'grouplist:foo', {}, [{ 'group:2', {}, [{ 'hitlist:hits', {prev=2}, [] }] }] }, " + + "{ 'grouplist:bar', {}, [{ 'group:4', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }"); + } + + @Test + public void requireThatUnstableContinuationsDoNotAffectSiblingHitLists() { + String request = "all(group(a) each(group(b) max(2) each(max(2) each(output(summary())))))"; + Grouping result = newGrouping(newGroup(2, 201, + newGroup(3, 301, newHitList(4, 4)), + newGroup(3, 302, newHitList(4, 4)), + newGroup(3, 303, newHitList(4, 4)), + newGroup(3, 304, newHitList(4, 4))), + newGroup(2, 202, + newGroup(3, 305, newHitList(4, 4)), + newGroup(3, 306, newHitList(4, 4)), + newGroup(3, 307, newHitList(4, 4)), + newGroup(3, 308, newHitList(4, 4)))); + assertContinuation(request, result, newComposite(), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:301', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:302', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:306', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:301', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:302', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:306', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2), + newOffset(newResultId(0, 0, 0), 2, 2)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:303', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:304', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=2}, [" + + "{ 'group:305', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:306', {}, [{ 'hitlist:hits', {prev=0}, [hit:3, hit:4] }] }] }] }] }] }"); + assertContinuation(request, result, newComposite(newOffset(newResultId(0, 1, 0, 1, 0), 4, 2), + newOffset(newResultId(0, 0, 0), 2, 2), + newUnstableOffset(newResultId(0, 1, 0), 2, 1)), + "{ 'group:root', {}, [{ 'grouplist:a', {}, [" + + "{ 'group:201', {}, [{ 'grouplist:b', {prev=0}, [" + + "{ 'group:303', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:304', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }, " + + "{ 'group:202', {}, [{ 'grouplist:b', {next=3, prev=0}, [" + + "{ 'group:306', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }, " + + "{ 'group:307', {}, [{ 'hitlist:hits', {next=2}, [hit:1, hit:2] }] }] }] }] }] }"); + } + + // -------------------------------------------------------------------------------- + // + // Utilities. + // + // -------------------------------------------------------------------------------- + + private static CompositeContinuation newComposite(EncodableContinuation... conts) { + CompositeContinuation ret = new CompositeContinuation(); + for (EncodableContinuation cont : conts) { + ret.add(cont); + } + return ret; + } + + private static ResultId newResultId(int... indexes) { + ResultId id = ResultId.valueOf(REQUEST_ID).newChildId(ROOT_IDX); + for (int i : indexes) { + id = id.newChildId(i); + } + return id; + } + + private static OffsetContinuation newOffset(ResultId resultId, int tag, int offset) { + return new OffsetContinuation(resultId, tag, offset, 0); + } + + private static OffsetContinuation newUnstableOffset(ResultId resultId, int tag, int offset) { + return new OffsetContinuation(resultId, tag, offset, OffsetContinuation.FLAG_UNSTABLE); + } + + private static Grouping newGrouping(Group... children) { + Group root = new Group(); + root.setTag(1); + for (Group child : children) { + root.addChild(child); + } + Grouping grouping = new Grouping(); + grouping.setRoot(root); + return grouping; + } + + private static Group newGroup(int tag, AggregationResult... results) { + return newGroup(tag, tag > 1 ? tag : 0, results); + } + + private static Group newGroup(int tag, int id, AggregationResult... results) { + Group group = new Group(); + group.setTag(tag); + if (id > 0) { + group.setId(new IntegerResultNode(id)); + } + for (AggregationResult result : results) { + group.addAggregationResult(result); + } + return group; + } + + private static Group newGroup(int tag, int id, Group child0, Group... childN) { + Group group = new Group(); + group.setTag(tag); + if (id > 0) { + group.setId(new IntegerResultNode(id)); + } + group.addChild(child0); + for (Group child : childN) { + group.addChild(child); + } + return group; + } + + private static HitsAggregationResult newHitList(int hitsTag, int numHits) { + HitsAggregationResult res = new HitsAggregationResult(); + res.setTag(hitsTag); + res.setSummaryClass("default"); + for (int i = 0; i < numHits; ++i) { + res.addHit(new FS4Hit(i + 1, new GlobalId(IdString.createIdString("doc:scheme:")), 1)); + } + return res; + } + + private static void assertGroupId(String expected, ResultNode actual) { + assertLayout("all(group(a) each(output(count())))", + newGrouping(new Group().setTag(2).setId(actual)), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=" + expected + "}[]]]"); + } + + private static void assertResult(String expected, AggregationResult actual) { + actual.setTag(3); + assertLayout("all(group(a) each(output(count())))", + newGrouping(newGroup(2, 2, actual)), + "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=" + expected + "}[]]]"); + } + + private static void assertBuild(String request, List<Grouping> result) { + ResultTest test = new ResultTest(); + test.result.addAll(result); + test.request = request; + assertOutput(test); + } + + private static void assertBuildFail(String request, List<Grouping> result, String expected) { + ResultTest test = new ResultTest(); + test.result.addAll(result); + test.request = request; + test.expectedException = expected; + assertOutput(test); + } + + private static void assertResultCont(String request, Grouping result, Continuation cont, String expected) { + assertOutput(request, Arrays.asList(result), cont, new ResultContWriter(), expected); + } + + private static void assertResultCont(String request, List<Grouping> result, String expected) { + assertOutput(request, result, null, new ResultContWriter(), expected); + } + + private static void assertResultCont(String request, List<Grouping> result, Continuation cont, String expected) { + assertOutput(request, result, cont, new ResultContWriter(), expected); + } + + private static void assertContinuation(String request, Grouping result, String expected) { + assertOutput(request, Arrays.asList(result), null, new ContinuationWriter(), expected); + } + + private static void assertContinuation(String request, Grouping result, Continuation cont, String expected) { + assertOutput(request, Arrays.asList(result), cont, new ContinuationWriter(), expected); + } + + private static void assertContinuation(String request, List<Grouping> result, Continuation cont, String expected) { + assertOutput(request, result, cont, new ContinuationWriter(), expected); + } + + private static void assertLayout(String request, Grouping result, String expected) { + assertOutput(request, Arrays.asList(result), null, new LayoutWriter(), expected); + } + + private static void assertLayout(String request, List<Grouping> result, String expected) { + assertOutput(request, result, null, new LayoutWriter(), expected); + } + + private static void assertOutput(String request, List<Grouping> result, Continuation continuation, + OutputWriter writer, String expected) { + ResultTest test = new ResultTest(); + test.result.addAll(result); + test.request = request; + test.outputWriter = writer; + test.continuation = continuation; + test.expectedOutput = expected; + assertOutput(test); + } + + private static void assertOutput(ResultTest test) { + RequestBuilder reqBuilder = new RequestBuilder(REQUEST_ID); + reqBuilder.setRootOperation(GroupingOperation.fromString(test.request)); + reqBuilder.addContinuations(Arrays.asList(test.continuation)); + reqBuilder.build(); + assertEquals(reqBuilder.getRequestList().size(), test.result.size()); + + ResultBuilder resBuilder = new ResultBuilder(); + resBuilder.setHitConverter(new MyHitConverter()); + resBuilder.setTransform(reqBuilder.getTransform()); + resBuilder.setRequestId(REQUEST_ID); + for (int i = 0, len = test.result.size(); i < len; ++i) { + Grouping grouping = test.result.get(i); + grouping.setId(i); + resBuilder.addGroupingResult(grouping); + } + try { + resBuilder.build(); + if (test.expectedException != null) { + fail("Expected exception '" + test.expectedException + "'."); + } + } catch (RuntimeException e) { + if (test.expectedException == null) { + throw e; + } + assertEquals(test.expectedException, e.getMessage()); + return; + } + if (test.outputWriter != null) { + String output = test.outputWriter.write(resBuilder); + assertEquals(test.expectedOutput, output); + } + } + + private static String getCanonicalId(com.yahoo.search.result.Hit hit) { + String str = hit.getId().toString(); + if (!str.startsWith("group:")) { + return str; + } + if (str.startsWith("group:root:")) { + return "group:root"; + } + int pos = str.indexOf(':', 6); + if (pos < 0) { + return str; + } + return "group:" + str.substring(pos + 1); + } + + private static class ResultTest { + + List<Grouping> result = new LinkedList<>(); + String request; + String expectedOutput; + String expectedException; + OutputWriter outputWriter; + Continuation continuation; + } + + private static interface OutputWriter { + + String write(ResultBuilder builder); + } + + private static class ResultContWriter implements OutputWriter { + + @Override + public String write(ResultBuilder builder) { + return toString(builder.getContinuation()); + } + + String toString(Continuation cnt) { + if (cnt instanceof OffsetContinuation) { + OffsetContinuation off = (OffsetContinuation)cnt; + String id = off.getResultId().toString().replace(", ", "."); + return id.substring(5, id.length() - 1) + "=" + off.getOffset(); + } else if (cnt instanceof CompositeContinuation) { + List<String> children = new LinkedList<>(); + for (Continuation child : (CompositeContinuation)cnt) { + children.add(toString(child)); + } + return children.toString(); + } else { + throw new UnsupportedOperationException(cnt.getClass().getName()); + } + } + } + + private static class ContinuationWriter implements OutputWriter { + + @Override + public String write(ResultBuilder builder) { + return toString(builder.getRoot()); + } + + String toString(com.yahoo.search.result.Hit hit) { + Map<String, String> conts = new TreeMap<>(); + if (hit instanceof AbstractList) { + for (Map.Entry<String, Continuation> entry : ((AbstractList)hit).continuations().entrySet()) { + conts.put(entry.getKey(), toString(entry.getValue())); + } + } + List<String> children = new LinkedList<>(); + if (hit instanceof HitGroup) { + for (com.yahoo.search.result.Hit childHit : (HitGroup)hit) { + if (childHit instanceof HitGroup) { + children.add(toString(childHit)); + } else { + children.add(childHit.getId().toString()); + } + } + } + return "{ '" + getCanonicalId(hit) + "', " + conts + ", " + children + " }"; + } + + String toString(Continuation cnt) { + if (cnt instanceof OffsetContinuation) { + return String.valueOf(((OffsetContinuation)cnt).getOffset()); + } else if (cnt instanceof CompositeContinuation) { + List<String> children = new LinkedList<>(); + for (Continuation child : (CompositeContinuation)cnt) { + children.add(toString(child)); + } + Collections.sort(children); + return children.toString(); + } else { + throw new UnsupportedOperationException(cnt.getClass().getName()); + } + } + } + + private static class LayoutWriter implements OutputWriter { + + @Override + public String write(ResultBuilder builder) { + return toString(builder.getRoot()); + } + + String toString(com.yahoo.search.result.Hit hit) { + StringBuilder ret = new StringBuilder(); + ret.append(hit.getClass().getSimpleName()); + + Map<String, String> members = new LinkedHashMap<>(); + if (hit instanceof GroupList) { + members.put("label", ((GroupList)hit).getLabel()); + } else if (hit instanceof HitList) { + members.put("label", ((HitList)hit).getLabel()); + } else { + members.put("id", getCanonicalId(hit)); + } + for (Map.Entry<String, Object> entry : hit.fields().entrySet()) { + members.put(entry.getKey(), String.valueOf(entry.getValue())); + } + ret.append(members); + + if (hit instanceof HitGroup) { + List<String> children = new LinkedList<>(); + for (com.yahoo.search.result.Hit childHit : (HitGroup)hit) { + children.add(toString(childHit)); + } + ret.append(children); + } + return ret.toString(); + } + } + + private static class MyHitConverter implements ResultBuilder.HitConverter { + + @Override + public com.yahoo.search.result.Hit toSearchHit(String summaryClass, com.yahoo.searchlib.aggregation.Hit hit) { + return new com.yahoo.search.result.Hit("hit:" + ((FS4Hit)hit).getPath(), new Relevance(0)); + } + } + + private static class MyAggregationResult extends AggregationResult { + + @Override + public ResultNode getRank() { + return null; + } + + @Override + protected void onMerge(AggregationResult result) { + + } + + @Override + protected boolean equalsAggregation(AggregationResult obj) { + return false; + } + } + + private static class MyResultNode extends ResultNode { + + @Override + protected void set(ResultNode rhs) { + + } + + @Override + protected int onCmp(ResultNode rhs) { + return 0; + } + + @Override + public long getInteger() { + return 0; + } + + @Override + public double getFloat() { + return 0; + } + + @Override + public String getString() { + return null; + } + + @Override + public byte[] getRaw() { + return new byte[0]; + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultIdTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultIdTestCase.java new file mode 100644 index 00000000000..8124b748f1f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultIdTestCase.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.grouping.vespa; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ResultIdTestCase { + + @Test + public void requireThatStartsWithWorks() { + assertFalse(ResultId.valueOf().startsWith(1, 1, 2, 3)); + assertFalse(ResultId.valueOf(1).startsWith(1, 1, 2, 3)); + assertFalse(ResultId.valueOf(1, 1).startsWith(1, 1, 2, 3)); + assertFalse(ResultId.valueOf(1, 1, 2).startsWith(1, 1, 2, 3)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith(1, 1, 2, 3)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith(1, 1, 2)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith(1, 1)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith(1)); + assertTrue(ResultId.valueOf(1, 1, 2, 3).startsWith()); + } + + @Test + public void requireThatChildIdStartsWithParentId() { + ResultId parentId = ResultId.valueOf(1, 1, 2); + ResultId childId = parentId.newChildId(3); + assertTrue(childId.startsWith(1, 1, 2)); + } + + @Test + public void requireThatHashCodeIsImplemented() { + assertEquals(ResultId.valueOf(1, 1, 2, 3).hashCode(), ResultId.valueOf(1, 1, 2, 3).hashCode()); + assertFalse(ResultId.valueOf(1, 1, 2, 3).hashCode() == ResultId.valueOf(5, 8, 13, 21).hashCode()); + } + + @Test + public void requireThatEqualsIsImplemented() { + assertEquals(ResultId.valueOf(1, 1, 2, 3), ResultId.valueOf(1, 1, 2, 3)); + assertFalse(ResultId.valueOf(1, 1, 2, 3).equals(ResultId.valueOf(5, 8, 13, 21))); + } + + @Test + public void requireThatResultIdCanBeEncoded() { + assertEncode("BIBCBCBEBG", ResultId.valueOf(1, 1, 2, 3)); + assertEncode("BIBKCBACBKCCK", ResultId.valueOf(5, 8, 13, 21)); + } + + @Test + public void requireThatResultIdCanBeDecoded() { + assertDecode(ResultId.valueOf(1, 1, 2, 3), "BIBCBCBEBG"); + assertDecode(ResultId.valueOf(5, 8, 13, 21), "BIBKCBACBKCCK"); + } + + private static void assertEncode(String expected, ResultId toEncode) { + IntegerEncoder encoder = new IntegerEncoder(); + toEncode.encode(encoder); + assertEquals(expected, encoder.toString()); + } + + private static void assertDecode(ResultId expected, String toDecode) { + IntegerDecoder decoder = new IntegerDecoder(toDecode); + assertTrue(decoder.hasNext()); + assertEquals(expected, ResultId.decode(decoder)); + assertFalse(decoder.hasNext()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java new file mode 100644 index 00000000000..d1cbf403c1a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/SearchHandlerTestCase.java @@ -0,0 +1,516 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.handler.test; + +import com.yahoo.container.Container; +import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; +import com.yahoo.container.jdisc.AsyncHttpResponse; +import com.yahoo.container.jdisc.HttpRequest; + +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.handler.RequestHandler; +import com.yahoo.processing.handler.ResponseStatus; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.handler.HttpSearchResponse; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.rendering.DefaultRenderer; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.concurrent.Executors; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author bratseth + */ +public class SearchHandlerTestCase { + + private static final String testDir = "src/test/java/com/yahoo/search/handler/test/config"; + private static String tempDir = ""; + private static String configId = null; + + @Rule + public TemporaryFolder tempfolder = new TemporaryFolder(); + + private RequestHandlerTestDriver driver = null; + private HandlersConfigurerTestWrapper configurer = null; + private SearchHandler searchHandler; + + @Before + public void startUp() throws IOException { + File cfgDir = tempfolder.newFolder("SearchHandlerTestCase"); + tempDir = cfgDir.getAbsolutePath(); + configId = "dir:" + tempDir; + + IOUtils.copyDirectory(new File(testDir), cfgDir, 1); // make configs active + generateComponentsConfigForActive(); + + configurer = new HandlersConfigurerTestWrapper(new Container(), configId); + searchHandler = (SearchHandler)configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + driver = new RequestHandlerTestDriver(searchHandler); + } + + @After + public void shutDown() { + if (configurer != null) configurer.shutdown(); + if (driver != null) driver.close(); + } + + private void generateComponentsConfigForActive() throws IOException { + File activeConfig = new File(tempDir); + SearchChainConfigurerTestCase. + createComponentsConfig(new File(activeConfig, "chains.cfg").getPath(), + new File(activeConfig, "handlers.cfg").getPath(), + new File(activeConfig, "components.cfg").getPath()); + } + + private SearchHandler fetchSearchHandler(HandlersConfigurerTestWrapper configurer) { + return (SearchHandler) configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + } + + @Test + public void testNullQuery() throws Exception { + assertEquals("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result total-hit-count=\"0\">\n" + + " <hit relevancy=\"1.0\">\n" + + " <field name=\"relevancy\">1.0</field>\n" + + " <field name=\"uri\">testHit</field>\n" + + " </hit>\n" + + "</result>\n", + driver.sendRequest("http://localhost?format=xml").readAll() + ); + } + + private String render(AsyncHttpResponse response) throws Exception { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + response.render(stream, null, null); + response.complete(); + return stream.toString(); + } + + @Test + public void testFailing() throws Exception { + assertTrue(driver.sendRequest("http://localhost?query=test&searchChain=classLoadingError").readAll().contains("NoClassDefFoundError")); + } + + @Test + public synchronized void testPluginError() throws Exception { + assertTrue(driver.sendRequest("http://localhost?query=test&searchChain=exceptionInPlugin").readAll().contains("NullPointerException")); + } + + @Test + public synchronized void testWorkingReconfiguration() throws Exception { + assertJsonResult("http://localhost?query=abc", driver); + + // reconfiguration + IOUtils.copyDirectory(new File(testDir, "handlers2"), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + + // ...and check the resulting config + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + assertTrue("Have a new instance of the search handler", searchHandler != newSearchHandler); + assertNotNull("Have the new search chain", fetchSearchHandler(configurer).getSearchChainRegistry().getChain("hello")); + assertNull("Don't have the new search chain", fetchSearchHandler(configurer).getSearchChainRegistry().getChain("classLoadingError")); + try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(searchHandler)) { + assertJsonResult("http://localhost?query=abc", newDriver); + } + } + + @Test + @Ignore //TODO: Must be done at the ConfiguredApplication level, not handlers configurer? Also, this must be rewritten as the above + public synchronized void testFailedReconfiguration() throws Exception { + assertXmlResult(driver); + + // attempt reconfiguration + IOUtils.copyDirectory(new File(testDir, "handlersInvalid"), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + RequestHandler newMockHandler = configurer.getRequestHandlerRegistry().getComponent("com.yahoo.search.handler.test.MockHandler"); + assertTrue("Reconfiguration failed: Kept the existing instance of the search handler", searchHandler == newSearchHandler); + assertNull("Reconfiguration failed: No mock handler", newMockHandler); + try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(searchHandler)) { + assertXmlResult(newDriver); + } + } + + @Test + public void testResponseBasics() { + Query q = new Query("?query=dummy&tracelevel=3"); + q.trace("nalle", 1); + Result r = new Result(q); + r.hits().addError(ErrorMessage.createUnspecifiedError("bamse")); + r.hits().add(new Hit("http://localhost/dummy", 0.5)); + HttpSearchResponse s = new HttpSearchResponse(200, r, q, new DefaultRenderer()); + assertEquals("text/xml", s.getContentType()); + assertNull(s.getCoverage()); + assertEquals("query 'dummy'", s.getParsedQuery()); + assertEquals(5000, s.getTiming().getTimeout()); + } + + @Test + public void testInvalidYqlQuery() throws Exception { + IOUtils.copyDirectory(new File(testDir, "config_yql"), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + assertTrue("Have a new instance of the search handler", searchHandler != newSearchHandler); + try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(newSearchHandler)) { + RequestHandlerTestDriver.MockResponseHandler responseHandler = newDriver.sendRequest( + "http://localhost/search/?yql=select%20*%20from%20foo%20where%20bar%20%3E%201453501295%27%3B"); + responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(400)); + } + } + + // Query handling takes a different code path when a query profile is active, so we test both paths. + @Test + public void testInvalidQueryParamWithQueryProfile() throws Exception { + try (RequestHandlerTestDriver newDriver = driverWithConfig("config_invalid_param")) { + testInvalidQueryParam(newDriver); + } + } + @Test + public void testInvalidQueryParamWithoutQueryProfile() throws Exception { + testInvalidQueryParam(driver); + } + private void testInvalidQueryParam(final RequestHandlerTestDriver testDriver) { + RequestHandlerTestDriver.MockResponseHandler responseHandler = + testDriver.sendRequest("http://localhost/search/?query=status_code%3A0&hits=20&offset=-20"); + String response = responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(400)); + assertThat(response, containsString("offset")); + assertThat(response, containsString("\"code\":" + com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code)); + } + + @Test + public void testWebServiceStatus() { + RequestHandlerTestDriver.MockResponseHandler responseHandler = + driver.sendRequest("http://localhost/search/?query=web_service_status_code"); + String response = responseHandler.readAll(); + assertThat(responseHandler.getStatus(), is(406)); + assertThat(response, containsString("\"code\":" + 406)); + } + + @Test + public void testNormalResultImplicitDefaultRendering() throws Exception { + assertJsonResult("http://localhost?query=abc", driver); + } + + @Test + public void testNormalResultExplicitDefaultRendering() throws Exception { + assertJsonResult("http://localhost?query=abc&format=default", driver); + } + + @Test + public void testNormalResultXmlAliasRendering() throws Exception { + assertXmlResult("http://localhost?query=abc&format=xml", driver); + } + + @Test + public void testNormalResultJsonAliasRendering() throws Exception { + assertJsonResult("http://localhost?query=abc&format=json", driver); + } + + @Test + public void testNormalResultExplicitDefaultRenderingFullRendererName1() throws Exception { + assertXmlResult("http://localhost?query=abc&format=DefaultRenderer", driver); + } + + @Test + public void testNormalResultExplicitDefaultRenderingFullRendererName2() throws Exception { + assertJsonResult("http://localhost?query=abc&format=JsonRenderer", driver); + } + + @Test + public void testResultLegacyTiledFormat() throws Exception { + assertTiledResult("http://localhost?query=abc&format=tiled", driver); + } + + @Test + public void testResultLegacyPageFormat() throws Exception { + assertPageResult("http://localhost?query=abc&format=page", driver); + } + + private static final String xmlResult = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result total-hit-count=\"0\">\n" + + " <hit relevancy=\"1.0\">\n" + + " <field name=\"relevancy\">1.0</field>\n" + + " <field name=\"uri\">testHit</field>\n" + + " </hit>\n" + + "</result>\n"; + private void assertXmlResult(String request, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(request), xmlResult); + } + private void assertXmlResult(RequestHandlerTestDriver driver) throws Exception { + assertXmlResult("http://localhost?query=abc", driver); + } + + private static final String jsonResult = "{\"root\":{" + + "\"id\":\"toplevel\",\"relevance\":1.0,\"fields\":{\"totalCount\":0}," + + "\"children\":[" + + "{\"id\":\"testHit\",\"relevance\":1.0,\"fields\":{\"uri\":\"testHit\"}}" + + "]}}"; + private void assertJsonResult(String request, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(request), jsonResult); + } + + private static final String tiledResult = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result version=\"1.0\">\n" + + "\n" + + " <hit relevance=\"1.0\">\n" + + " <id>testHit</id>\n" + + " <uri>testHit</uri>\n" + + " </hit>\n" + + "\n" + + "</result>\n"; + private void assertTiledResult(String request, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(request), tiledResult); + } + + private static final String pageResult = + "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<page version=\"1.0\">\n" + + "\n" + + " <content>\n" + + " <hit relevance=\"1.0\">\n" + + " <id>testHit</id>\n" + + " <uri>testHit</uri>\n" + + " </hit>\n" + + " </content>\n" + + "\n" + + "</page>\n"; + private void assertPageResult(String request, RequestHandlerTestDriver driver) throws Exception { + assertOkResult(driver.sendRequest(request), pageResult); + } + + private void assertOkResult(RequestHandlerTestDriver.MockResponseHandler response, String expected) { + assertEquals(expected, response.readAll()); + assertEquals(200, response.getStatus()); + } + + @Test + public void testFaultyHandlers() throws Exception { + assertHandlerResponse(500, null, "NullReturning"); + assertHandlerResponse(500, null, "NullReturningAsync"); + assertHandlerResponse(500, null, "Throwing"); + assertHandlerResponse(500, null, "ThrowingAsync"); + } + + @Test + public void testForwardingHandlers() throws Exception { + assertHandlerResponse(200, jsonResult, "ForwardingAsync"); + + // Fails because we are forwarding from a sync to an async handler - + // the sync handler will respond with status 500 because the async one has not produced a response yet. + // Disabled because this fails due to a race and is therefore unstable + // assertHandlerResponse(500, null, "Forwarding"); + } + + private void assertHandlerResponse(int status, String responseData, String handlerName) throws Exception { + RequestHandler forwardingHandler = configurer.getRequestHandlerRegistry().getComponent("com.yahoo.search.handler.test.SearchHandlerTestCase$" + handlerName + "Handler"); + try (RequestHandlerTestDriver forwardingDriver = new RequestHandlerTestDriver(forwardingHandler)) { + RequestHandlerTestDriver.MockResponseHandler response = forwardingDriver.sendRequest("http://localhost/" + handlerName + "?query=test"); + response.awaitResponse(); + assertEquals("Expected HTTP status", status, response.getStatus()); + if (responseData == null) + assertEquals("Connection closed with no data", null, response.read()); + else + assertEquals(responseData, response.readAll()); + } + } + + private RequestHandlerTestDriver driverWithConfig(String configDirectory) throws Exception { + IOUtils.copyDirectory(new File(testDir, configDirectory), new File(tempDir), 1); + generateComponentsConfigForActive(); + configurer.reloadConfig(); + + SearchHandler newSearchHandler = fetchSearchHandler(configurer); + assertTrue("Should have a new instance of the search handler", searchHandler != newSearchHandler); + return new RequestHandlerTestDriver(newSearchHandler); + } + + /** Referenced from config */ + public static class TestSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = new Result(query); + Hit hit = new Hit("testHit"); + hit.setField("uri", "testHit"); + result.hits().add(hit); + + if (query.getModel().getQueryString().contains("web_service_status_code")) + result.hits().addError(new ErrorMessage(406, "Test web service code")); + + return result; + } + } + + /** Referenced from config */ + public static class ClassLoadingErrorSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + throw new NoClassDefFoundError(); // Simulate typical OSGi problem + } + } + + /** Referenced from config */ + public static class ExceptionInPluginSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + try { + result.hits().add(null); // Trigger NullPointerException + } catch (NullPointerException e) { + throw new RuntimeException("Message", e); + } + return result; + } + } + + /** Referenced from config */ + public static class HelloWorldSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(new Hit("HelloWorld")); + return result; + } + } + + /** Referenced from config */ + public static class ForwardingHandler extends ThreadedHttpRequestHandler { + + private final SearchHandler searchHandler; + + public ForwardingHandler(SearchHandler searchHandler) { + super(Executors.newSingleThreadExecutor(), null, false); + this.searchHandler = searchHandler; + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + try { + HttpRequest forwardRequest = + new HttpRequest.Builder(httpRequest).uri(new URI("http://localhost/search/?query=test")).createDirectRequest(); + return searchHandler.handle(forwardRequest); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + } + + /** Referenced from config */ + public static class ForwardingAsyncHandler extends ThreadedHttpRequestHandler { + + private final SearchHandler searchHandler; + + public ForwardingAsyncHandler(SearchHandler searchHandler) { + super(Executors.newSingleThreadExecutor(), null, true); + this.searchHandler = searchHandler; + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + try { + HttpRequest forwardRequest = + new HttpRequest.Builder(httpRequest).uri(new URI("http://localhost/search/?query=test")).createDirectRequest(); + return searchHandler.handle(forwardRequest); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + } + + /** Referenced from config */ + public static class NullReturningHandler extends ThreadedHttpRequestHandler { + + public NullReturningHandler() { + super(Executors.newSingleThreadExecutor(), null, false); + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + return null; + } + + } + + /** Referenced from config */ + public static class NullReturningAsyncHandler extends ThreadedHttpRequestHandler { + + public NullReturningAsyncHandler() { + super(Executors.newSingleThreadExecutor(), null, true); + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + return null; + } + + } + + /** Referenced from config */ + public static class ThrowingHandler extends ThreadedHttpRequestHandler { + + public ThrowingHandler() { + super(Executors.newSingleThreadExecutor(), null, false); + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + throw new RuntimeException(); + } + + } + + /** Referenced from config */ + public static class ThrowingAsyncHandler extends ThreadedHttpRequestHandler { + + public ThrowingAsyncHandler() { + super(Executors.newSingleThreadExecutor(), null, true); + } + + @Override + public HttpResponse handle(HttpRequest httpRequest) { + throw new RuntimeException(); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/.gitignore b/container-search/src/test/java/com/yahoo/search/handler/test/config/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/.gitignore diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg new file mode 100644 index 00000000000..0336e06f54b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/chains.cfg @@ -0,0 +1,14 @@ +chains[3] +chains[0].id default +chains[0].components[1] +chains[0].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +chains[1].id classLoadingError +chains[1].components[1] +chains[1].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$ClassLoadingErrorSearcher +chains[2].id exceptionInPlugin +chains[2].components[1] +chains[2].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$ExceptionInPluginSearcher +components[3] +components[0].id com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +components[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$ClassLoadingErrorSearcher +components[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$ExceptionInPluginSearcher diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/config_invalid_param/query-profiles.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/config_invalid_param/query-profiles.cfg new file mode 100644 index 00000000000..a1009f05310 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/config_invalid_param/query-profiles.cfg @@ -0,0 +1,5 @@ +queryprofile[1] +queryprofile[0].id "default" +queryprofile[0].property[1] +queryprofile[0].property[0].name "maxOffset" +queryprofile[0].property[0].value "1000" diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/config_yql/chains.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/config_yql/chains.cfg new file mode 100644 index 00000000000..72c1af55bc5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/config_yql/chains.cfg @@ -0,0 +1,6 @@ +chains[1] +chains[0].id default +chains[0].components[1] +chains[0].components[0] com.yahoo.search.yql.MinimalQueryInserter +components[1] +components[0].id com.yahoo.search.yql.MinimalQueryInserter diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg new file mode 100644 index 00000000000..96843d78aae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers.cfg @@ -0,0 +1,8 @@ +handler[7] +handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningHandler +handler[2].id com.yahoo.search.handler.test.SearchHandlerTestCase$NullReturningAsyncHandler +handler[3].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingHandler +handler[4].id com.yahoo.search.handler.test.SearchHandlerTestCase$ThrowingAsyncHandler +handler[5].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingHandler +handler[6].id com.yahoo.search.handler.test.SearchHandlerTestCase$ForwardingAsyncHandler diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg new file mode 100644 index 00000000000..2437efdec4f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlers2/chains.cfg @@ -0,0 +1,10 @@ +chains[2] +chains[0].id default +chains[0].components[1] +chains[0].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +chains[1].id hello +chains[1].components[1] +chains[1].components[0] com.yahoo.search.handler.test.SearchHandlerTestCase$HelloWorldSearcher +components[2] +components[0].id com.yahoo.search.handler.test.SearchHandlerTestCase$TestSearcher +components[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$HelloWorldSearcher diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg new file mode 100644 index 00000000000..691b37b4955 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/handlersInvalid/handlers.cfg @@ -0,0 +1,3 @@ +handler[2] +handler[0].id com.yahoo.search.handler.SearchHandler +handler[1].id com.yahoo.search.handler.test.SearchHandlerTestCase$ErrorOnInitializationHandler diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/index-info.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/index-info.cfg diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-search.cfg diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/config/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/handler/test/config/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/test/config/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/match/test/DocumentDbTest.java b/container-search/src/test/java/com/yahoo/search/match/test/DocumentDbTest.java new file mode 100644 index 00000000000..feddcbde505 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/match/test/DocumentDbTest.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.match.test; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.search.Result; +import com.yahoo.search.match.DocumentDb; +import com.yahoo.text.MapParser; +import org.junit.Ignore; +import org.junit.Test; +import static org.junit.Assert.*; + +public class DocumentDbTest { + + @Test + @Ignore + public void testWand() { + DocumentDb db = new DocumentDb(); + db.put(createFeatureDocument("1","[a:7, b:5, c:3]")); + db.put(createFeatureDocument("2", "[a:2, b:1, c:4]")); + //Result r = execute(createWandQuery("[a:1, b:3, c:5]")); + //assertEquals(67,r.hits().get(0).getRelevance()); + //assertEquals(25, r.hits().get(1).getRelevance()); + } + + private DocumentPut createFeatureDocument(String localId, String features) { + DocumentType type = new DocumentType("withFeature"); + type.addField("features", new WeightedSetDataType(DataType.STRING,true,true)); + Document d = new Document(type, new DocumentId("id:test::" + localId)); + d.setFieldValue("features",fillFromString(new WeightedSet(DataType.STRING), features)); + return new DocumentPut(d); + } + + // TODO: Move to weightedset + // TODO: Don't pass through a map + private WeightedSet fillFromString(WeightedSet s, String values) { + //new IntMapParser().parse(); + return null; + } + + private static class IntMapParser extends MapParser<Integer> { + + @Override + protected Integer parseValue(String s) { + return Integer.parseInt(s); + } + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/MapPageTemplateXMLReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/MapPageTemplateXMLReadingTestCase.java new file mode 100644 index 00000000000..052ea62d4f0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/MapPageTemplateXMLReadingTestCase.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.config.test; + +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.PageTemplateRegistry; +import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; +import com.yahoo.search.pagetemplates.model.*; + +import java.util.List; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class MapPageTemplateXMLReadingTestCase extends junit.framework.TestCase { + + private String root="src/test/java/com/yahoo/search/pagetemplates/config/test/examples/mapexamples/"; + + public void testMap1() { + PageTemplateRegistry registry=new PageTemplateXMLReader().read(root); + assertCorrectMap1(registry.getComponent("map1")); + } + + private void assertCorrectMap1(PageTemplate page) { + assertNotNull("map1 was read",page); + Section root=page.getSection(); + assertTrue(((Section)((Section)root.elements(Section.class).get(0)).elements(Section.class).get(0)).elements().get(0) instanceof Placeholder); + assertTrue(((Section)((Section)root.elements(Section.class).get(0)).elements(Section.class).get(1)).elements().get(0) instanceof Placeholder); + assertTrue(((Section)((Section)root.elements(Section.class).get(1)).elements(Section.class).get(0)).elements().get(0) instanceof Placeholder); + assertTrue(((Section)((Section)root.elements(Section.class).get(1)).elements(Section.class).get(1)).elements().get(0) instanceof Placeholder); + assertEquals("box1source",((Placeholder) ((Section)((Section)root.elements(Section.class).get(0)).elements(Section.class).get(0)).elements().get(0)).getId()); + assertEquals("box2source",((Placeholder) ((Section)((Section)root.elements(Section.class).get(0)).elements(Section.class).get(1)).elements().get(0)).getId()); + assertEquals("box3source",((Placeholder) ((Section)((Section)root.elements(Section.class).get(1)).elements(Section.class).get(0)).elements().get(0)).getId()); + assertEquals("box4source",((Placeholder) ((Section)((Section)root.elements(Section.class).get(1)).elements(Section.class).get(1)).elements().get(0)).getId()); + + MapChoice map=(MapChoice)root.elements().get(2); + assertEquals("box1source",map.placeholderIds().get(0)); + assertEquals("box2source",map.placeholderIds().get(1)); + assertEquals("box3source",map.placeholderIds().get(2)); + assertEquals("box4source",map.placeholderIds().get(3)); + assertEquals("source1",((Source)((List<?>)map.values().get(0)).get(0)).getName()); + assertEquals("source2",((Source)((List<?>)map.values().get(1)).get(0)).getName()); + assertEquals("source3",((Source)((List<?>)map.values().get(2)).get(0)).getName()); + assertEquals("source4",((Source)((List<?>)map.values().get(3)).get(0)).getName()); + + PageTemplateXMLReadingTestCase.assertCorrectSources("source1 source2 source3 source4",page); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/PageTemplateXMLReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/PageTemplateXMLReadingTestCase.java new file mode 100644 index 00000000000..7832719412a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/PageTemplateXMLReadingTestCase.java @@ -0,0 +1,279 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.config.test; + +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.PageTemplateRegistry; +import com.yahoo.search.pagetemplates.PageTemplatesConfig; +import com.yahoo.search.pagetemplates.config.PageTemplateConfigurer; +import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.Renderer; +import com.yahoo.search.pagetemplates.model.Section; +import com.yahoo.search.pagetemplates.model.Source; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class PageTemplateXMLReadingTestCase extends junit.framework.TestCase { + + private String root="src/test/java/com/yahoo/search/pagetemplates/config/test/"; + + public void testExamples() { + PageTemplateRegistry registry=new PageTemplateXMLReader().read(root + "examples"); + assertCorrectSerp(registry.getComponent("serp")); + assertCorrectSlottingSerp(registry.getComponent("slottingSerp")); + assertCorrectRichSerp(registry.getComponent("richSerp")); + assertCorrectRicherSerp(registry.getComponent("richerSerp")); + assertCorrectIncluder(registry.getComponent("includer")); + assertCorrectGeneric(registry.getComponent("generic")); + } + + public void testConfigReading() { + PageTemplatesConfig config = new PageTemplatesConfig(new PageTemplatesConfig.Builder() + .page("<page id=\"slottingSerp\" layout=\"mainAndRight\">\n <section layout=\"column\" region=\"main\" source=\"*\" order=\"-[rank]\"/>\n <section layout=\"column\" region=\"right\" source=\"ads\"/>\n</page>\n") + .page("<page id=\"richSerp\" layout=\"mainAndRight\">\n <section layout=\"row\" region=\"main\">\n <section layout=\"column\" description=\"left main pane\">\n <section layout=\"row\" max=\"5\" description=\"Bar of images, from one of two possible sources\">\n <choice method=\"annealing\">\n <source name=\"images\"/>\n <source name=\"flickr\"/>\n </choice>\n </section>\n <section max=\"1\" source=\"local map video ticker weather\" description=\"A single relevant graphically rich element\"/>\n <section order=\"-[rank]\" max=\"10\" source=\"web news\" description=\"Various kinds of traditional search results\"/>\n </section>\n <section layout=\"column\" order=\"[source]\" source=\"answers blogs twitter\" description=\"right main pane, ugc stuff, grouped by source\"/>\n </section>\n <section layout=\"column\" source=\"ads\" region=\"right\"/>\n</page>\n") + .page("<page id=\"footer\">\n <section layout=\"row\" source=\"popularSearches\"/>\n <section id=\"extraFooter\" layout=\"row\" source=\"topArticles\"/>\n</page>\n") + .page("<page id=\"richerSerp\" layout=\"column\">\n <include idref=\"header\"/>\n <section layout=\"mainAndRight\">\n <section layout=\"row\" region=\"main\">\n <section layout=\"column\" description=\"left main pane\">\n <choice>\n <alternative>\n <section layout=\"row\" max=\"5\" description=\"Bar of images, from one of two possible sources\">\n <choice>\n <source name=\"images\"/>\n <alternative>\n <source name=\"flickr\">\n <renderer name=\"mouseOverImage\"/>\n </source>\n <source name=\"twitpic\">\n <choice>\n <renderer name=\"mouseOverImage\">\n <parameter name=\"hovertime\">5</parameter>\n <parameter name=\"borderColor\">#ff00ff</parameter>\n </renderer>\n <renderer name=\"regularImage\"/>\n </choice>\n <parameter name=\"filter\">origin=twitter</parameter>\n </source>\n </alternative>\n </choice>\n <choice>\n <renderer name=\"regularImageBox\"/>\n <renderer name=\"newImageBox\"/>\n </choice>\n </section>\n <section max=\"1\" source=\"local map video ticker weather\" description=\"A single relevant graphically rich element\"/>\n </alternative>\n <section order=\"[source]\" max=\"10\" source=\"web news\" description=\"Various kinds of traditional search results\"/>\n </choice>\n </section>\n <section layout=\"column\" order=\"[source]\" source=\"answers blogs twitter\" description=\"right main pane, ugc stuff, grouped by source\"/>\n </section>\n <section layout=\"column\" source=\"ads\" region=\"right\" order=\"-[rank] clickProbability\">\n <renderer name=\"newAdBox\"/>\n </section>\n </section>\n <include idref=\"footer\"/>\n</page>\n") + .page("<page id=\"header\">\n <section layout=\"row\">\n <section source=\"global\"/>\n <section source=\"notifications\"/>\n </section>\n</page>\n") + ); + PageTemplateRegistry registry = PageTemplateConfigurer.toRegistry(config); + assertCorrectSlottingSerp(registry.getComponent("slottingSerp")); + assertCorrectRichSerp(registry.getComponent("richSerp")); + assertCorrectRicherSerp(registry.getComponent("richerSerp")); + } + + public void testInvalidFilename() { + try { + PageTemplateRegistry registry=new PageTemplateXMLReader().read(root + "examples/invalidfilename"); + assertEquals(0,registry.allComponents().size()); + fail("Should have caused an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("The file name of page template 'notinvalid' must be 'notinvalid.xml' but was 'invalid.xml'",e.getMessage()); + } + } + + protected void assertCorrectSerp(PageTemplate page) { + assertNotNull("'serp' was read",page); + Section rootSection=page.getSection(); + assertNotNull(rootSection); + assertEquals("mainAndRight",rootSection.getLayout().getName()); + Section main=(Section)rootSection.elements(Section.class).get(0); + assertEquals("column",main.getLayout().getName()); + assertEquals("main",main.getRegion()); + assertEquals("web",((Source)main.elements(Source.class).get(0)).getName()); + Section right=(Section)rootSection.elements(Section.class).get(1); + assertEquals("column",right.getLayout().getName()); + assertEquals("right",right.getRegion()); + assertEquals("ads",((Source)right.elements(Source.class).get(0)).getName()); + } + + protected void assertCorrectSlottingSerp(PageTemplate page) { + assertNotNull("'slotting serp' was read",page); + Section rootSection=page.getSection(); + Section main=(Section)rootSection.elements(Section.class).get(0); + assertEquals("-[rank]",main.getOrder().toString()); + assertEquals(Source.any,main.elements(Source.class).get(0)); + + assertCorrectSources("* ads",page); + } + + protected void assertCorrectRichSerp(PageTemplate page) { + assertNotNull("'rich serp' was read",page); + Section rootSection=page.getSection(); + assertNotNull(rootSection); + assertEquals("mainAndRight",rootSection.getLayout().getName()); + + Section main=(Section)rootSection.elements(Section.class).get(0); + assertEquals("row",main.getLayout().getName()); + assertEquals("main",main.getRegion()); + Section leftMain=(Section)main.elements(Section.class).get(0); + assertEquals("column",leftMain.getLayout().getName()); + Section imageBar=(Section)leftMain.elements(Section.class).get(0); + assertEquals("row",imageBar.getLayout().getName()); + assertEquals(5,imageBar.getMax()); + assertEquals("annealing",((Choice)imageBar.elements(Source.class).get(0)).getMethod().toString()); + assertEquals("images",((Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(0).get(0)).getName()); + assertEquals("flickr",((Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(1).get(0)).getName()); + Section richElement=(Section)leftMain.elements(Section.class).get(1); + assertEquals(1,richElement.getMax()); + assertEquals("[source 'local', source 'map', source 'video', source 'ticker', source 'weather']",richElement.elements(Source.class).toString()); + Section webResults=(Section)leftMain.elements(Section.class).get(2); + assertEquals("-[rank]",webResults.getOrder().toString()); + assertEquals(10,webResults.getMax()); + assertEquals("[source 'web', source 'news']",webResults.elements(Source.class).toString()); + Section rightMain=(Section)main.elements(Section.class).get(1); + assertEquals("column",rightMain.getLayout().getName()); + assertEquals("+[source]",rightMain.getOrder().toString()); + assertEquals("[source 'answers', source 'blogs', source 'twitter']",rightMain.elements(Source.class).toString()); + + Section right=(Section)rootSection.elements(Section.class).get(1); + assertEquals("column",right.getLayout().getName()); + assertEquals("right",right.getRegion()); + assertEquals("ads",((Source)right.elements(Source.class).get(0)).getName()); + } + + protected void assertCorrectRicherSerp(PageTemplate page) { + assertNotNull("'richer serp' was read",page); + + // Check resolution as we go + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(Choice.createSingleton(page),null,null); + + Section root=page.getSection(); + assertNotNull(root); + assertEquals("column",root.getLayout().getName()); + + assertEquals("Sections was correctly imported and combined with the section in this",4,root.elements(Section.class).size()); + + assertCorrectHeader((Section)root.elements(Section.class).get(0)); + + Section body=(Section)root.elements(Section.class).get(1); + assertEquals("mainAndRight",body.getLayout().getName()); + + Section main=(Section)body.elements(Section.class).get(0); + assertEquals("row",main.getLayout().getName()); + assertEquals("main",main.getRegion()); + + Section leftMain=(Section)main.elements(Section.class).get(0); + assertEquals("column",leftMain.getLayout().getName()); + assertEquals(1,resolution.getResolution((Choice)leftMain.elements(Section.class).get(0))); + + Section imageBar=(Section)((Choice)leftMain.elements(Section.class).get(0)).alternatives().get(0).get(0); + assertEquals("row",imageBar.getLayout().getName()); + assertEquals(5,imageBar.getMax()); + assertEquals(2,((Choice)imageBar.elements(Source.class).get(0)).alternatives().size()); + assertEquals("images",((Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(0).get(0)).getName()); + assertEquals(1,resolution.getResolution((Choice)imageBar.elements(Source.class).get(0))); + assertEquals(1,resolution.getResolution((Choice)imageBar.elements(Renderer.class).get(0))); + + Source flickrSource=(Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(1).get(0); + assertEquals("flickr",flickrSource.getName()); + assertEquals(1,flickrSource.renderers().size()); + assertEquals("mouseOverImage",((Renderer)flickrSource.renderers().get(0)).getName()); + + Source twitpicSource=(Source)((Choice)imageBar.elements(Source.class).get(0)).alternatives().get(1).get(1); + assertEquals("twitpic",twitpicSource.getName()); + assertEquals(1,twitpicSource.parameters().size()); + assertEquals("origin=twitter",twitpicSource.parameters().get("filter")); + assertEquals(2,((Choice)twitpicSource.renderers().get(0)).alternatives().size()); + assertEquals(1,resolution.getResolution((Choice)twitpicSource.renderers().get(0))); + + Renderer mouseOverImageRenderer=(Renderer)((Choice)twitpicSource.renderers().get(0)).alternatives().get(0).get(0); + assertEquals("mouseOverImage", mouseOverImageRenderer.getName()); + assertEquals(2, mouseOverImageRenderer.parameters().size()); + assertEquals("5", mouseOverImageRenderer.parameters().get("hovertime")); + assertEquals("#ff00ff", mouseOverImageRenderer.parameters().get("borderColor")); + assertEquals("regularImage",((Renderer)((Choice)twitpicSource.renderers().get(0)).alternatives().get(1).get(0)).getName()); + assertEquals(2,((Choice)imageBar.elements(Renderer.class).get(0)).alternatives().size()); + assertEquals("regularImageBox",((Renderer)((Choice)imageBar.elements(Renderer.class).get(0)).alternatives().get(0).get(0)).getName()); + assertEquals("newImageBox",((Renderer)((Choice)imageBar.elements(Renderer.class).get(0)).alternatives().get(1).get(0)).getName()); + + Section richElement=(Section)((Choice)leftMain.elements(Section.class).get(0)).get(0).get(1); + assertEquals(1,richElement.getMax()); + assertEquals("[source 'local', source 'map', source 'video', source 'ticker', source 'weather']",richElement.elements(Source.class).toString()); + + Section webResults=(Section)((Choice)leftMain.elements(Section.class).get(0)).get(1).get(0); + assertEquals("+[source]",webResults.getOrder().toString()); + assertEquals(10,webResults.getMax()); + assertEquals("[source 'web', source 'news']",webResults.elements(Source.class).toString()); + + Section rightMain=(Section)main.elements(Section.class).get(1); + assertEquals("column",rightMain.getLayout().getName()); + assertEquals("+[source]",rightMain.getOrder().toString()); + assertEquals("[source 'answers', source 'blogs', source 'twitter']",rightMain.elements(Source.class).toString()); + + Section right=(Section)body.elements(Section.class).get(1); + assertEquals("column",right.getLayout().getName()); + assertEquals("right",right.getRegion()); + assertEquals("ads",((Source)right.elements(Source.class).get(0)).getName()); + assertEquals("newAdBox",((Renderer)right.elements(Renderer.class).get(0)).getName()); + assertEquals("-[rank] +clickProbability",right.getOrder().toString()); + + assertCorrectFooter((Section)root.elements(Section.class).get(2)); + assertEquals("extraFooter",((Section)root.elements(Section.class).get(3)).getId()); + + // Check getSources() + assertCorrectSources("ads answers blogs flickr global images local map news notifications " + + "popularSearches ticker topArticles twitpic twitter video weather web",page); + } + + static void assertCorrectSources(String expectedSourceNameString,PageTemplate page) { + String[] expectedSourceNames=expectedSourceNameString.split(" "); + Set<String> sourceNames=new HashSet<>(); + for (Source source : page.getSources()) + sourceNames.add(source.getName()); + assertEquals("Expected " + expectedSourceNames.length + " elements in " + sourceNames, + expectedSourceNames.length,sourceNames.size()); + for (String expectedSourceName : expectedSourceNames) + assertTrue("Sources did not include '" + expectedSourceName+ "'",sourceNames.contains(expectedSourceName)); + } + + protected void assertCorrectIncluder(PageTemplate page) { + assertNotNull("'includer' was read",page); + + Resolution resolution=new DeterministicResolver().resolve(Choice.createSingleton(page),null,null); + + Section case1=(Section)page.getSection().elements(Section.class).get(0); + assertCorrectHeader((Section)case1.elements(Section.class).get(0)); + assertCorrectFooter((Section)case1.elements(Section.class).get(1)); + + Section case2=(Section)page.getSection().elements(Section.class).get(1); + assertCorrectHeader((Section)((Choice)case2.elements(Section.class).get(0)).get(0).get(0)); + assertCorrectFooter((Section)((Choice)case2.elements(Section.class).get(0)).get(1).get(0)); + assertEquals(1,resolution.getResolution((Choice)case2.elements(Section.class).get(0))); + + Section case3=(Section)page.getSection().elements(Section.class).get(2); + assertCorrectHeader((Section)((Choice)case3.elements(Section.class).get(0)).get(0).get(0)); + assertCorrectFooter((Section)((Choice)case3.elements(Section.class).get(0)).get(1).get(0)); + assertEquals(1,resolution.getResolution((Choice)case3.elements(Section.class).get(0))); + + Section case4=(Section)page.getSection().elements(Section.class).get(3); + assertEquals("first",((Section)((Choice)case4.elements(Section.class).get(0)).get(0).get(0)).getId()); + assertCorrectHeader((Section)((Choice)case4.elements(Section.class).get(0)).get(1).get(0)); + assertEquals("middle",((Section)((Choice)case4.elements(Section.class).get(0)).get(2).get(0)).getId()); + assertCorrectFooter((Section)((Choice)case4.elements(Section.class).get(0)).get(3).get(0)); + assertEquals("last",((Section)((Choice)case4.elements(Section.class).get(0)).get(4).get(0)).getId()); + assertEquals(4,resolution.getResolution((Choice)case4.elements(Section.class).get(0))); + + Section case5=(Section)page.getSection().elements(Section.class).get(4); + assertEquals(2,((Choice)case5.elements(Section.class).get(0)).alternatives().size()); + assertCorrectHeader((Section)((Choice)case5.elements(Section.class).get(0)).get(0).get(0)); + assertEquals("second",((Section)((Choice)case5.elements(Section.class).get(0)).get(1).get(0)).getId()); + assertEquals(1,resolution.getResolution((Choice)case5.elements(Section.class).get(0))); + + // This case - a choice inside a choice - makes little sense. It is included as a reminder - + // what we really want is to be able to include some additional alternatives of a choice, + // but without any magic. That requires allowing alternative as a top-level tag, or something + Section case6=(Section)page.getSection().elements(Section.class).get(5); + Choice includerChoice=(Choice)case6.elements().get(0); + Choice includedChoice=(Choice)includerChoice.alternatives().get(0).get(0); + assertCorrectFooter((Section)includedChoice.alternatives().get(0).get(0)); + } + + private void assertCorrectHeader(Section header) { + assertEquals("row",header.getLayout().getName()); + assertEquals(2,header.elements(Section.class).size()); + assertEquals( "global",((Source)((Section)header.elements(Section.class).get(0)).elements(Source.class).get(0)).getName()); + assertEquals("notifications",((Source)((Section)header.elements(Section.class).get(1)).elements(Source.class).get(0)).getName()); + } + + private void assertCorrectFooter(Section footer) { + assertEquals("row",footer.getLayout().getName()); + assertTrue(footer.elements(Section.class).isEmpty()); + assertEquals("popularSearches",((Source)footer.elements(Source.class).get(0)).getName()); + } + + private void assertCorrectGeneric(PageTemplate page) { + assertEquals("image", ((Source)((Section)page.getSection().elements(Section.class).get(0)).elements(Source.class).get(0)).getName()); + assertEquals("flickr", ((Source)((Section)page.getSection().elements(Section.class).get(0)).elements(Source.class).get(1)).getName()); + assertEquals(Source.any,((Section)page.getSection().elements(Section.class).get(1)).elements(Source.class).get(0)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceFooter.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceFooter.xml new file mode 100644 index 00000000000..9ebdaeb9302 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceFooter.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="choiceFooter"> + <choice> + <section layout="row" source="popularSearches"/> + </choice> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceHeader.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceHeader.xml new file mode 100644 index 00000000000..36b0ae6430c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/choiceHeader.xml @@ -0,0 +1,10 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="choiceHeader"> + <choice> + <section layout="row"> + <section source="global"/> + <section source="notifications"/> + </section> + <section id="second" source="blog"/> + </choice> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/footer.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/footer.xml new file mode 100644 index 00000000000..0866aaaa583 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/footer.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="footer"> + <section layout="row" source="popularSearches"/> + <section id="extraFooter" layout="row" source="topArticles"/> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/generic.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/generic.xml new file mode 100644 index 00000000000..319f3058d24 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/generic.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="generic"> + <section source="image flickr"/> + <section source="*"/> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/header.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/header.xml new file mode 100644 index 00000000000..a894e8b9a3e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/header.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="header"> + <section layout="row"> + <section source="global"/> + <section source="notifications"/> + </section> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/includer.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/includer.xml new file mode 100644 index 00000000000..6d4f6121991 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/includer.xml @@ -0,0 +1,36 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="includer" description="Demonstrates the various include cases"> + <section id="case1" description="No choices"> + <include idref="header"/> + <include idref="footer"/> + </section> + <section id="case2" description="Include as implicit alternatives"> + <choice> + <include idref="header"/> + <include idref="footer"/> + </choice> + </section> + <section id="case3" description="Include as explicit alternatives - same result as above"> + <choice> + <alternative><include idref="header"/></alternative> + <alternative><include idref="footer"/></alternative> + </choice> + </section> + <section id="case4" description="Mixed with un-included"> + <choice> + <section id="first" source="music"/> + <alternative><include idref="header"/></alternative> + <section id="middle" source="video"/> + <alternative><include idref="footer"/></alternative> + <section id="last" source="books"/> + </choice> + </section> + <section id="case5" description="Including two alternatives"> + <include idref="choiceHeader"/> + </section> + <section id="case6" description="Including one choice"> + <choice> + <include idref="choiceFooter"/> + </choice> + </section> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/invalidfilename/invalid.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/invalidfilename/invalid.xml new file mode 100644 index 00000000000..0e799a472de --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/invalidfilename/invalid.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="notinvalid"> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/mapexamples/map1.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/mapexamples/map1.xml new file mode 100644 index 00000000000..c13fdcdbbca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/mapexamples/map1.xml @@ -0,0 +1,21 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="map1" layout="column" description="Contains 4 boxes, to which 4 sources may be added"> + + <section layout="row" description="row 1"> + <section id="box1"><placeholder id="box1source"/></section> + <section id="box2"><placeholder id="box2source"/></section> + </section> + <section layout="row" description="row 2"> + <section id="box3"><placeholder id="box3source"/></section> + <section id="box4"><placeholder id="box4source"/></section> + </section> + + <choice> + <map to="box1source box2source box3source box4source"> + <source name="source1"/> + <source name="source2"/> + <source name="source3"/> + <source name="source4"/> + </map> + </choice> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richSerp.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richSerp.xml new file mode 100644 index 00000000000..32ab6086b82 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richSerp.xml @@ -0,0 +1,17 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="richSerp" layout="mainAndRight"> + <section layout="row" region="main"> + <section layout="column" description="left main pane"> + <section layout="row" max="5" description="Bar of images, from one of two possible sources"> + <choice method="annealing"> + <source name="images"/> + <source name="flickr"/> + </choice> + </section> + <section max="1" source="local map video ticker weather" description="A single relevant graphically rich element"/> + <section order="-[rank]" max="10" source="web news" description="Various kinds of traditional search results"/> + </section> + <section layout="column" order="[source]" source="answers blogs twitter" description="right main pane, ugc stuff, grouped by source"/> + </section> + <section layout="column" source="ads" region="right"/> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richerSerp.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richerSerp.xml new file mode 100644 index 00000000000..d3e11288ef1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/richerSerp.xml @@ -0,0 +1,45 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="richerSerp" layout="column"> + <include idref="header"/> + <section layout="mainAndRight"> + <section layout="row" region="main"> + <section layout="column" description="left main pane"> + <choice> + <alternative> + <section layout="row" max="5" description="Bar of images, from one of two possible sources"> + <choice> + <source name="images"/> + <alternative> + <source name="flickr"> + <renderer name="mouseOverImage"/> + </source> + <source name="twitpic"> + <choice> + <renderer name="mouseOverImage"> + <parameter name="hovertime">5</parameter> + <parameter name="borderColor">#ff00ff</parameter> + </renderer> + <renderer name="regularImage"/> + </choice> + <parameter name="filter">origin=twitter</parameter> + </source> + </alternative> + </choice> + <choice> + <renderer name="regularImageBox"/> + <renderer name="newImageBox"/> + </choice> + </section> + <section max="1" source="local map video ticker weather" description="A single relevant graphically rich element"/> + </alternative> + <section order="[source]" max="10" source="web news" description="Various kinds of traditional search results"/> + </choice> + </section> + <section layout="column" order="[source]" source="answers blogs twitter" description="right main pane, ugc stuff, grouped by source"/> + </section> + <section layout="column" source="ads" region="right" order="-[rank] clickProbability"> + <renderer name="newAdBox"/> + </section> + </section> + <include idref="footer"/> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/serp.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/serp.xml new file mode 100644 index 00000000000..194c551f84c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/serp.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="serp" layout="mainAndRight"> + <section layout="column" region="main" source="web"/> + <section layout="column" region="right" source="ads"/> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/slottingSerp.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/slottingSerp.xml new file mode 100644 index 00000000000..301f7e77edb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/config/test/examples/slottingSerp.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="slottingSerp" layout="mainAndRight"> + <section layout="column" region="main" source="*" order="-[rank]"/> + <section layout="column" region="right" source="ads"/> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySource.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySource.xml new file mode 100644 index 00000000000..4a5b6b3a1dd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySource.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="AnySource" source="source3 *"> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceResult.xml new file mode 100644 index 00000000000..40cc6b6935e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceResult.xml @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result version="1.0"> + + <hit relevance="1.0" source="source3"> + <id>source3-1</id> + </hit> + + <hit relevance="1.0" source="source1"> + <id>source1-1</id> + </hit> + + <hit relevance="1.0" source="source2"> + <id>source2-1</id> + </hit> + + <hit relevance="0.5" source="source3"> + <id>source3-2</id> + </hit> + + <hit relevance="0.5" source="source1"> + <id>source1-2</id> + </hit> + + <hit relevance="0.5" source="source2"> + <id>source2-2</id> + </hit> + + <hit relevance="0.3333333333333333" source="source3"> + <id>source3-3</id> + </hit> + + <hit relevance="0.3333333333333333" source="source1"> + <id>source1-3</id> + </hit> + + <hit relevance="0.3333333333333333" source="source2"> + <id>source2-3</id> + </hit> + +</result> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceTestCase.java new file mode 100644 index 00000000000..dc20e5483ca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/AnySourceTestCase.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class AnySourceTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("AnySource.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",3)); + result.hits().add(createHits("source3",3)); + + // Resolve (noop here) + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check execution: + // all three sources, ordered by relevance, source 3 first in each relevance group + HitGroup hits=result.hits(); + assertEquals(9,hits.size()); + assertEquals("source3-1",hits.get(0).getId().stringValue()); + assertEquals("source1-1",hits.get(1).getId().stringValue()); + assertEquals("source2-1",hits.get(2).getId().stringValue()); + assertEquals("source3-2",hits.get(3).getId().stringValue()); + assertEquals("source1-2",hits.get(4).getId().stringValue()); + assertEquals("source2-2",hits.get(5).getId().stringValue()); + assertEquals("source3-3",hits.get(6).getId().stringValue()); + assertEquals("source1-3",hits.get(7).getId().stringValue()); + assertEquals("source2-3",hits.get(8).getId().stringValue()); + + // Check rendering + assertRendered(result,"AnySourceResult.xml"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderers.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderers.xml new file mode 100644 index 00000000000..e0306872149 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderers.xml @@ -0,0 +1,17 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="ChoiceOfRenderers" source="source1 source2"> + + <choice> + <renderer name="sectionLook1"/> + <renderer name="sectionLook2"/> + </choice> + <choice> + <renderer name="source1Look1" for="source1"/> + <renderer name="source1Look2" for="source1"/> + <renderer name="source1Look3" for="source1"> + <parameter name="color">#ff00ff</parameter> + <parameter name="blink">true</parameter> + </renderer> + </choice> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersResult.xml new file mode 100644 index 00000000000..47ed1bd2f12 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersResult.xml @@ -0,0 +1,36 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result version="1.0"> + + <renderer name="sectionLook2"/> + + <renderer for="source1" name="source1Look3"> + <parameter name="color">#ff00ff</parameter> + <parameter name="blink">true</parameter> + </renderer> + + <hit relevance="1.0" source="source1"> + <id>source1-1</id> + </hit> + + <hit relevance="1.0" source="source2"> + <id>source2-1</id> + </hit> + + <hit relevance="0.5" source="source1"> + <id>source1-2</id> + </hit> + + <hit relevance="0.5" source="source2"> + <id>source2-2</id> + </hit> + + <hit relevance="0.3333333333333333" source="source1"> + <id>source1-3</id> + </hit> + + <hit relevance="0.3333333333333333" source="source2"> + <id>source2-3</id> + </hit> + +</result> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersTestCase.java new file mode 100644 index 00000000000..58d05971805 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfRenderersTestCase.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.Renderer; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ChoiceOfRenderersTestCase extends ExecutionAbstractTestCase { + + //This test is order dependent. Fix this!! + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("ChoiceOfRenderers.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",3)); + result.hits().add(createHits("source3",3)); + + // Resolve + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + assertEquals(1,resolution.getResolution((Choice)((PageTemplate)page.get(0).get(0)).getSection().elements(Renderer.class).get(0))); + assertEquals(2,resolution.getResolution((Choice)((PageTemplate)page.get(0).get(0)).getSection().elements(Renderer.class).get(1))); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + assertEquals(6,result.getConcreteHitCount()); + assertEquals(6,result.getHitCount()); + + // Check rendering + assertRendered(result,"ChoiceOfRenderersResult.xml"); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsections.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsections.xml new file mode 100644 index 00000000000..f7323ba094d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsections.xml @@ -0,0 +1,20 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="ChoiceOfSubsections"> + <choice method="method1"> + <alternative> + <section source="source0"/> + </alternative> + <alternative> + <section source="source1"/> + </alternative> + <alternative> + <section source="source2"/> + <section> + <choice method="method2"> + <source name="source3"/> + <source name="source4"/> + </choice> + </section> + </alternative> + </choice> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsResult.xml new file mode 100644 index 00000000000..b1d29995312 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsResult.xml @@ -0,0 +1,29 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result version="1.0"> + + <section> + <hit relevance="1.0" source="source2"> + <id>source2-1</id> + </hit> + <hit relevance="0.5" source="source2"> + <id>source2-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source2"> + <id>source2-3</id> + </hit> + </section> + + <section> + <hit relevance="1.0" source="source4"> + <id>source4-1</id> + </hit> + <hit relevance="0.5" source="source4"> + <id>source4-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source4"> + <id>source4-3</id> + </hit> + </section> + +</result> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsTestCase.java new file mode 100644 index 00000000000..3d92a721f0d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfSubsectionsTestCase.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ChoiceOfSubsectionsTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("ChoiceOfSubsections.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",3)); + result.hits().add(createHits("source3",3)); + result.hits().add(createHits("source4",3)); + + new Organizer().organize(page,new DeterministicResolverAssertingMethod().resolve(page,query,result),result); + + // Check execution: + // Two subsections with one source each + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertEqualHitGroups(createHits("source2",3),section1); + assertEqualHitGroups(createHits("source4",3),section2); + + // Check rendering + assertRendered(result,"ChoiceOfSubsectionsResult.xml"); + } + + /** Same as deterministic resolver, but asserts that it received the correct method names for each choice */ + private static class DeterministicResolverAssertingMethod extends DeterministicResolver { + + private int invocationNumber=0; + + /** Chooses the last alternative of any choice */ + @Override + public void resolve(Choice choice, Query query, Result result, Resolution resolution) { + invocationNumber++; + if (invocationNumber==2) + assertEquals("method1",choice.getMethod()); + else if (invocationNumber==3) + assertEquals("method2",choice.getMethod()); + else if (invocationNumber>3) + throw new IllegalStateException("Unexpected number of resolver invocations"); + + super.resolve(choice,query,result,resolution); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSources.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSources.xml new file mode 100644 index 00000000000..c6a0af9ddd2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSources.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="ChoiceOfTwoSources"> + <choice> + <source name="source1"/> + <source name="source2"/> + </choice> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesResult.xml new file mode 100644 index 00000000000..35d913fd9fa --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesResult.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result version="1.0"> + + <hit relevance="1.0" source="source2"> + <id>source2-1</id> + </hit> + + <hit relevance="0.5" source="source2"> + <id>source2-2</id> + </hit> + + <hit relevance="0.3333333333333333" source="source2"> + <id>source2-3</id> + </hit> + +</result> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesTestCase.java new file mode 100644 index 00000000000..facffb50649 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoiceOfTwoSourcesTestCase.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.Source; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ChoiceOfTwoSourcesTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("ChoiceOfTwoSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",3)); + result.hits().add(createHits("source3",3)); + + // Resolve + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + assertEquals(1,resolution.getResolution((Choice)((PageTemplate)page.get(0).get(0)).getSection().elements(Source.class).get(0))); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check execution: + // No subsections: Last choice (source2) used + assertEqualHitGroups(createHits("source2",3),result.hits()); + + // Check rendering + assertRendered(result,"ChoiceOfTwoSourcesResult.xml"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Choices.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Choices.xml new file mode 100644 index 00000000000..e8d1736f46c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Choices.xml @@ -0,0 +1,45 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="Choices"> + <choice> + + <alternative> + <section layout="row"> + <section id="realtime"> + <choice> + <source name="news"/> + <source name="blog"/> + </choice> + </section> + <section source="images" max="2" id="multimedia"/> + <section source="web" id="web"/> + </section> + </alternative> + + <alternative> + <section source="*" id="blended"/> + </alternative> + + <alternative> + <section layout="row" description="row 1"> + <section id="box1"><placeholder id="box1source"/></section> + <section id="box2"><placeholder id="box2source"/></section> + </section> + <section layout="row" description="row 2"> + <section id="box3"><placeholder id="box3source"/></section> + <section id="box4"><placeholder id="box4source"/></section> + </section> + + <choice method="myMethod"> + <map to="box1source box2source box3source box4source"> + <source name="news"/> + <source name="web"/> + <source name="blog"/> + <source name="images"/> + </map> + </choice> + + </alternative> + + </choice> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesResult.xml new file mode 100644 index 00000000000..ab995365f22 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesResult.xml @@ -0,0 +1,55 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result version="1.0"> + + <section layout="row"> + <section id="section:box1"> + <hit relevance="1.0" source="news"> + <id>news-1</id> + </hit> + <hit relevance="0.5" source="news"> + <id>news-2</id> + </hit> + <hit relevance="0.3333333333333333" source="news"> + <id>news-3</id> + </hit> + </section> + <section id="section:box2"> + <hit relevance="1.0" source="web"> + <id>web-1</id> + </hit> + <hit relevance="0.5" source="web"> + <id>web-2</id> + </hit> + <hit relevance="0.3333333333333333" source="web"> + <id>web-3</id> + </hit> + </section> + </section> + + <section layout="row"> + <section id="section:box3"> + <hit relevance="1.0" source="blog"> + <id>blog-1</id> + </hit> + <hit relevance="0.5" source="blog"> + <id>blog-2</id> + </hit> + <hit relevance="0.3333333333333333" source="blog"> + <id>blog-3</id> + </hit> + </section> + <section id="section:box4"> + <hit relevance="1.0" source="images"> + <id>images-1</id> + </hit> + <hit relevance="0.5" source="images"> + <id>images-2</id> + </hit> + <hit relevance="0.3333333333333333" source="images"> + <id>images-3</id> + </hit> + </section> + </section> + +</result> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesTestCase.java new file mode 100644 index 00000000000..a646823c8cb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ChoicesTestCase.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.PageElement; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ChoicesTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template (second alternative will be chosen) + List<PageElement> pages=new ArrayList<>(); + pages.add(importPage("AnySource.xml")); + pages.add(importPage("Choices.xml")); + Choice page=Choice.createSingletons(pages); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("news",3)); + result.hits().add(createHits("web",3)); + result.hits().add(createHits("blog",3)); + result.hits().add(createHits("images",3)); + + // Resolve + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"ChoicesResult.xml"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java new file mode 100644 index 00000000000..544366758f3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.io.IOUtils; +import com.yahoo.prelude.templates.TiledTemplateSet; +import com.yahoo.prelude.templates.UserTemplate; +import com.yahoo.prelude.templates.test.TilingTestCase; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +import java.io.*; + +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class ExecutionAbstractTestCase { + + private static final String root="src/test/java/com/yahoo/search/pagetemplates/engine/test/"; + + protected PageTemplate importPage(String name) { + PageTemplate template=new PageTemplateXMLReader().readFile(root + name); + assertNotNull("Could look up page template '" + name + "'",template); + return template; + } + + protected void assertEqualHitGroups(HitGroup expected,HitGroup actual) { + assertEquals(expected.size(),actual.size()); + int i=0; + for (Hit expectedHit : expected.asList()) { + Hit actualHit=actual.get(i++); + assertEquals(expectedHit.getId(),actualHit.getId()); + assertEquals(expectedHit.getSource(),actualHit.getSource()); + } + } + + protected HitGroup createHits(String sourceName,int hitCount) { + HitGroup source=new HitGroup("source:" + sourceName); + for (int i=1; i<=hitCount; i++) { + Hit hit=new Hit(sourceName + "-" + i,1/(double)i); + hit.setSource(sourceName); + source.add(hit); + } + return source; + } + + protected void assertRendered(Result result,String resultFileName) { + assertRendered(result,resultFileName,false); + } + + protected void assertRendered(Result result,String resultFileName, UserTemplate<?> template) { + assertRendered(result,resultFileName,template,false); + } + + protected void assertRendered(Result result,String resultFileName,boolean print) { + assertRendered(result,resultFileName,new TiledTemplateSet(),print); + } + + @SuppressWarnings("deprecation") + protected void assertRendered(Result result,String resultFileName,UserTemplate<?> template, boolean print) { + result.getTemplating().setTemplates(template); + try { + TilingTestCase.assertRendered(IOUtils.readFile(new File(root + resultFileName)), result); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSections.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSections.xml new file mode 100644 index 00000000000..2bc75fba5f4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSections.xml @@ -0,0 +1,28 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="MapSectionsToSections" layout="column" description="Contains 4 boxes, to which 4 sections are mapped"> + + <section layout="row" description="row 1"> + <placeholder id="box1holder"/> + <placeholder id="box2holder"/> + </section> + <section layout="row" description="row 2"> + <placeholder id="box3holder"/> + <placeholder id="box4holder"/> + </section> + + <choice method="myMethod"> + <map to="box1holder box2holder box3holder box4holder"> + <section id="box1" source="source1"/> + <section id="box2" source="source2"/> + <item> + <section id="box3" source="source3"/> + <section id="box5" source="source5"/> + </item> + <section id="box4" source="source4"/> + </map> + </choice> + + <choice> <!-- Empty choices should have no effect --> + </choice> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsResult.xml new file mode 100644 index 00000000000..3a163e5f804 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsResult.xml @@ -0,0 +1,96 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result version="1.0" layout="column"> + + <section layout="row"> + <section id="section:box1"> + <hit relevance="1.0" source="source1"> + <id>source1-1</id> + </hit> + <hit relevance="0.5" source="source1"> + <id>source1-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source1"> + <id>source1-3</id> + </hit> + </section> + <section id="section:box2"> + <hit relevance="1.0" source="source2"> + <id>source2-1</id> + </hit> + <hit relevance="0.5" source="source2"> + <id>source2-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source2"> + <id>source2-3</id> + </hit> + <hit relevance="0.25" source="source2"> + <id>source2-4</id> + </hit> + </section> + </section> + + <section layout="row"> + <section id="section:box3"> + <hit relevance="1.0" source="source3"> + <id>source3-1</id> + </hit> + <hit relevance="0.5" source="source3"> + <id>source3-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source3"> + <id>source3-3</id> + </hit> + <hit relevance="0.25" source="source3"> + <id>source3-4</id> + </hit> + <hit relevance="0.2" source="source3"> + <id>source3-5</id> + </hit> + </section> + <section id="section:box5"> + <hit relevance="1.0" source="source5"> + <id>source5-1</id> + </hit> + <hit relevance="0.5" source="source5"> + <id>source5-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source5"> + <id>source5-3</id> + </hit> + <hit relevance="0.25" source="source5"> + <id>source5-4</id> + </hit> + <hit relevance="0.2" source="source5"> + <id>source5-5</id> + </hit> + <hit relevance="0.1666666666666667" source="source5"> + <id>source5-6</id> + </hit> + <hit relevance="0.1428571428571428" source="source5"> + <id>source5-7</id> + </hit> + </section> + <section id="section:box4"> + <hit relevance="1.0" source="source4"> + <id>source4-1</id> + </hit> + <hit relevance="0.5" source="source4"> + <id>source4-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source4"> + <id>source4-3</id> + </hit> + <hit relevance="0.25" source="source4"> + <id>source4-4</id> + </hit> + <hit relevance="0.2" source="source4"> + <id>source4-5</id> + </hit> + <hit relevance="0.1666666666666667" source="source4"> + <id>source4-6</id> + </hit> + </section> + </section> + +</result> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsTestCase.java new file mode 100644 index 00000000000..54fc342aa22 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSectionsToSectionsTestCase.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.MapChoice; +import com.yahoo.search.pagetemplates.model.PageElement; +import com.yahoo.search.pagetemplates.model.Section; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class MapSectionsToSectionsTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("MapSectionsToSections.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",4)); + result.hits().add(createHits("source3",5)); + result.hits().add(createHits("source4",6)); + result.hits().add(createHits("source5",7)); + + // Resolve + Resolver resolver=new DeterministicResolverAssertingMethod(); + Resolution resolution=resolver.resolve(page,query,result); + Map<String, List<PageElement>> mapping= + resolution.getResolution((MapChoice)((PageTemplate)page.get(0).get(0)).getSection().elements().get(2)); + assertNotNull(mapping); + assertEquals("box1",((Section)mapping.get("box1holder").get(0)).getId()); + assertEquals("box2",((Section)mapping.get("box2holder").get(0)).getId()); + assertEquals("box3",((Section)mapping.get("box3holder").get(0)).getId()); + assertEquals("box4",((Section)mapping.get("box4holder").get(0)).getId()); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check execution: + // Two subsections, each containing two sub-subsections with one source each + assertEquals(2,result.hits().size()); + HitGroup row1=(HitGroup)result.hits().get(0); + HitGroup column11=(HitGroup)row1.get(0); + HitGroup column12=(HitGroup)row1.get(1); + HitGroup row2=(HitGroup)result.hits().get(1); + HitGroup column21a=(HitGroup)row2.get(0); + HitGroup column21b=(HitGroup)row2.get(1); + HitGroup column22=(HitGroup)row2.get(2); + assertEqualHitGroups(createHits("source1",3),column11); + assertEqualHitGroups(createHits("source2",4),column12); + assertEqualHitGroups(createHits("source3",5),column21a); + assertEqualHitGroups(createHits("source5",7),column21b); + assertEqualHitGroups(createHits("source4",6),column22); + + // Check rendering + assertRendered(result,"MapSectionsToSectionsResult.xml"); + } + + /** Same as deterministic resolver, but asserts that it received the correct method names for each map choice */ + private static class DeterministicResolverAssertingMethod extends DeterministicResolver { + + /** Chooses the last alternative of any choice */ + @Override + public void resolve(MapChoice mapChoice, Query query, Result result, Resolution resolution) { + assertEquals("myMethod",mapChoice.getMethod()); + super.resolve(mapChoice,query,result,resolution); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSections.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSections.xml new file mode 100644 index 00000000000..7b5c0770096 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSections.xml @@ -0,0 +1,22 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="MapSourcesToSections" layout="column" description="4 sources are assigned to a section each"> + + <section layout="row" description="row 1"> + <section id="box1"><placeholder id="box1source"/></section> + <section id="box2"><placeholder id="box2source"/></section> + </section> + <section layout="row" description="row 2"> + <section id="box3"><placeholder id="box3source"/></section> + <section id="box4"><placeholder id="box4source"/></section> + </section> + + <choice method="myMethod"> + <map to="box1source box2source box3source box4source"> + <source name="source1"/> + <source name="source2"/> + <source name="source3"/> + <source name="source4"/> + </map> + </choice> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsResult.xml new file mode 100644 index 00000000000..034330c071c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsResult.xml @@ -0,0 +1,73 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result version="1.0" layout="column"> + + <section layout="row"> + <section id="section:box1"> + <hit relevance="1.0" source="source1"> + <id>source1-1</id> + </hit> + <hit relevance="0.5" source="source1"> + <id>source1-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source1"> + <id>source1-3</id> + </hit> + </section> + <section id="section:box2"> + <hit relevance="1.0" source="source2"> + <id>source2-1</id> + </hit> + <hit relevance="0.5" source="source2"> + <id>source2-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source2"> + <id>source2-3</id> + </hit> + <hit relevance="0.25" source="source2"> + <id>source2-4</id> + </hit> + </section> + </section> + + <section layout="row"> + <section id="section:box3"> + <hit relevance="1.0" source="source3"> + <id>source3-1</id> + </hit> + <hit relevance="0.5" source="source3"> + <id>source3-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source3"> + <id>source3-3</id> + </hit> + <hit relevance="0.25" source="source3"> + <id>source3-4</id> + </hit> + <hit relevance="0.2" source="source3"> + <id>source3-5</id> + </hit> + </section> + <section id="section:box4"> + <hit relevance="1.0" source="source4"> + <id>source4-1</id> + </hit> + <hit relevance="0.5" source="source4"> + <id>source4-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source4"> + <id>source4-3</id> + </hit> + <hit relevance="0.25" source="source4"> + <id>source4-4</id> + </hit> + <hit relevance="0.2" source="source4"> + <id>source4-5</id> + </hit> + <hit relevance="0.1666666666666667" source="source4"> + <id>source4-6</id> + </hit> + </section> + </section> + +</result> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsTestCase.java new file mode 100644 index 00000000000..49cc0411ac5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/MapSourcesToSectionsTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.MapChoice; +import com.yahoo.search.pagetemplates.model.PageElement; +import com.yahoo.search.pagetemplates.model.Source; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class MapSourcesToSectionsTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("MapSourcesToSections.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",4)); + result.hits().add(createHits("source3",5)); + result.hits().add(createHits("source4",6)); + result.hits().add(createHits("source5",7)); + + // Resolve + Resolver resolver=new DeterministicResolverAssertingMethod(); + Resolution resolution=resolver.resolve(page,query,result); + Map<String, List<PageElement>> mapping= + resolution.getResolution((MapChoice)((PageTemplate)page.get(0).get(0)).getSection().elements().get(2)); + assertNotNull(mapping); + assertEquals("source1",((Source)mapping.get("box1source").get(0)).getName()); + assertEquals("source2",((Source)mapping.get("box2source").get(0)).getName()); + assertEquals("source3",((Source)mapping.get("box3source").get(0)).getName()); + assertEquals("source4",((Source)mapping.get("box4source").get(0)).getName()); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check execution: + // Two subsections, each containing two sub-subsections with one source each + assertEquals(2,result.hits().size()); + HitGroup row1=(HitGroup)result.hits().get(0); + HitGroup column11=(HitGroup)row1.get(0); + HitGroup column12=(HitGroup)row1.get(1); + HitGroup row2=(HitGroup)result.hits().get(1); + HitGroup column21=(HitGroup)row2.get(0); + HitGroup column22=(HitGroup)row2.get(1); + assertEqualHitGroups(createHits("source1",3),column11); + assertEqualHitGroups(createHits("source2",4),column12); + assertEqualHitGroups(createHits("source3",5),column21); + assertEqualHitGroups(createHits("source4",6),column22); + + // Check rendering + assertRendered(result,"MapSourcesToSectionsResult.xml"); + } + + /** Same as deterministic resolver, but asserts that it received the correct method names for each map choice */ + private static class DeterministicResolverAssertingMethod extends DeterministicResolver { + + /** Chooses the last alternative of any choice */ + @Override + public void resolve(MapChoice mapChoice, Query query, Result result, Resolution resolution) { + assertEquals("myMethod",mapChoice.getMethod()); + super.resolve(mapChoice,query,result,resolution); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Page.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Page.xml new file mode 100644 index 00000000000..967da527b6e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/Page.xml @@ -0,0 +1,31 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="Page"> + + <renderer name="two-column"/> + + <section region="left"> + <source url="http://carmot.yahoo.com:4080/resource/[news article id]"/> + <renderer name="articleBodyRenderer"> + <parameter name="color">blue</parameter> + </renderer> + </section> + + <section region="right"> + <renderer name="multi-item-column"> + <parameter name="items">3</parameter> + </renderer> + + <section region="1" source="news"> + <renderer name="articleListRenderer"/> + </section> + + <section region="2"> + <source url="http://vitality.yahoo.com:4080/consumption-widget"/> + <renderer name="identityRenderer"/> + </section> + + <section region="3" source="htmlSource"> + <renderer name="htmlRenderer"/> + </section> + </section> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageResult.xml new file mode 100644 index 00000000000..95b86ef1f4d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageResult.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page version="1.0"> + + <renderer name="two-column"/> + + <section region="left"> + <source url="http://carmot.yahoo.com:4080/resource/[news article id]"/> + <renderer name="articleBodyRenderer"> + <parameter name="color">blue</parameter> + </renderer> + </section> + + <section region="right"> + <renderer name="multi-item-column"> + <parameter name="items">3</parameter> + </renderer> + <section region="1"> + <renderer name="articleListRenderer"/> + <content> + <hit relevance="1.0" source="news"> + <id>news-1</id> + </hit> + <hit relevance="0.5" source="news"> + <id>news-2</id> + </hit> + </content> + </section> + <section region="2"> + <source url="http://vitality.yahoo.com:4080/consumption-widget"/> + <renderer name="identityRenderer"/> + </section> + <section region="3"> + <renderer name="htmlRenderer"/> + <content> + <hit relevance="1.0" source="htmlSource"> + <id>htmlSource-1</id> + </hit> + </content> + </section> + </section> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageTestCase.java new file mode 100644 index 00000000000..33930f3d0e0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.prelude.templates.PageTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import org.junit.Test; + +/** + * Tests an example page. + * + * @author bratseth + */ +public class PageTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("Page.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("news",2)); + result.hits().add(createHits("htmlSource",1)); + + // Resolve (noop here) + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"PageResult.xml",new PageTemplateSet()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlending.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlending.xml new file mode 100644 index 00000000000..dca31eb42e1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlending.xml @@ -0,0 +1,37 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="PageWithBlending"> + + <renderer name="two-column"/> + + <section region="left"> + <source url="http://carmot.yahoo.com:4080/resource/[news article id]"/> + <renderer name="articleBodyRenderer"> + <parameter name="color">blue</parameter> + </renderer> + </section> + + <section region="right"> + <renderer name="multi-item-column"> + <parameter name="items">3</parameter> + </renderer> + + <section region="1"> + <renderer for="newsImage" name="newsImageRenderer"/> + <source name="news"> + <renderer name="articleRenderer"/> + </source> + <source name="image"> + <renderer name="imageRenderer"/> + </source> + </section> + + <section region="2"> + <source url="http://vitality.yahoo.com:4080/consumption-widget"/> + <renderer name="identityRenderer"/> + </section> + + <section region="3" source="htmlSource"> + <renderer name="htmlRenderer"/> + </section> + </section> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingResult.xml new file mode 100644 index 00000000000..7ac78f3e820 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingResult.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page version="1.0"> + + <renderer name="two-column"/> + + <section region="left"> + <source url="http://carmot.yahoo.com:4080/resource/[news article id]"/> + <renderer name="articleBodyRenderer"> + <parameter name="color">blue</parameter> + </renderer> + </section> + + <section region="right"> + <renderer name="multi-item-column"> + <parameter name="items">3</parameter> + </renderer> + <section region="1"> + <renderer for="newsImage" name="newsImageRenderer"/> + <renderer for="news" name="articleRenderer"/> + <renderer for="image" name="imageRenderer"/> + <content> + <hit relevance="1.0" source="news"> + <id>news-1</id> + </hit> + <hit relevance="0.5" source="news"> + <id>news-2</id> + </hit> + </content> + </section> + <section region="2"> + <source url="http://vitality.yahoo.com:4080/consumption-widget"/> + <renderer name="identityRenderer"/> + </section> + <section region="3"> + <renderer name="htmlRenderer"/> + <content> + <hit relevance="1.0" source="htmlSource"> + <id>htmlSource-1</id> + </hit> + </content> + </section> + </section> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingTestCase.java new file mode 100644 index 00000000000..445105cfd2f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithBlendingTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.prelude.templates.PageTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import org.junit.Test; + +/** + * Tests an exapnded example. + * + * @author bratseth + */ +public class PageWithBlendingTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("PageWithBlending.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("news",2)); + result.hits().add(createHits("htmlSource",1)); + + // Resolve (noop here) + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"PageWithBlendingResult.xml",new PageTemplateSet()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRenderer.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRenderer.xml new file mode 100644 index 00000000000..5d3e38c2beb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRenderer.xml @@ -0,0 +1,36 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="PageWithSourceRenderer"> + + <renderer name="two-column"/> + + <section region="left"> + <choice> + <source url="notchosen"/> + <source url="http://carmot.yahoo.com:4080/resource/[news article id]"/> + </choice> + <renderer name="articleBodyRenderer"> + <parameter name="color">blue</parameter> + </renderer> + </section> + + <section region="right"> + <renderer name="multi-item-column"> + <parameter name="items">3</parameter> + </renderer> + + <section region="1"> + <source name="news"> + <renderer name="articleRenderer"/> + </source> + </section> + + <section region="2"> + <source url="http://vitality.yahoo.com:4080/consumption-widget"/> + <renderer name="identityRenderer"/> + </section> + + <section region="3" source="htmlSource"> + <renderer name="htmlRenderer"/> + </section> + </section> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererResult.xml new file mode 100644 index 00000000000..00656399331 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererResult.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page version="1.0"> + + <renderer name="two-column"/> + + <section region="left"> + <source url="http://carmot.yahoo.com:4080/resource/[news article id]"/> + <renderer name="articleBodyRenderer"> + <parameter name="color">blue</parameter> + </renderer> + </section> + + <section region="right"> + <renderer name="multi-item-column"> + <parameter name="items">3</parameter> + </renderer> + <section region="1"> + <renderer for="news" name="articleRenderer"/> + <content> + <hit relevance="1.0" source="news"> + <id>news-1</id> + </hit> + <hit relevance="0.5" source="news"> + <id>news-2</id> + </hit> + </content> + </section> + <section region="2"> + <source url="http://vitality.yahoo.com:4080/consumption-widget"/> + <renderer name="identityRenderer"/> + </section> + <section region="3"> + <renderer name="htmlRenderer"/> + <content> + <hit relevance="1.0" source="htmlSource"> + <id>htmlSource-1</id> + </hit> + </content> + </section> + </section> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererTestCase.java new file mode 100644 index 00000000000..e7dbae21d47 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/PageWithSourceRendererTestCase.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.prelude.templates.PageTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import org.junit.Test; + +/** + * Tests an example with two data sources with a renderer each. + * + * @author bratseth + */ +public class PageWithSourceRendererTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("PageWithSourceRenderer.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("news",2)); + result.hits().add(createHits("htmlSource",1)); + + // Resolve + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"PageWithSourceRendererResult.xml",new PageTemplateSet()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoice.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoice.xml new file mode 100644 index 00000000000..ff39ef1d3d1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoice.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="SourceChoice"> + <choice> + <source name="news"/> + <source name="web"/> + </choice> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceResult.xml new file mode 100644 index 00000000000..c9e0909a476 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceResult.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page version="1.0"> + + <content> + <hit relevance="1.0" source="web"> + <id>web-1</id> + </hit> + <hit relevance="0.5" source="web"> + <id>web-2</id> + </hit> + <hit relevance="0.3333333333333333" source="web"> + <id>web-3</id> + </hit> + </content> + +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceTestCase.java new file mode 100644 index 00000000000..04e550a631c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/SourceChoiceTestCase.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.prelude.templates.PageTemplateSet; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.Resolver; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import org.junit.Test; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class SourceChoiceTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("SourceChoice.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("web",3)); + result.hits().add(createHits("news",3)); + result.hits().add(createHits("blog",3)); + + // Resolve (noop here) + Resolver resolver=new DeterministicResolver(); + Resolution resolution=resolver.resolve(page,query,result); + + // Execute + Organizer organizer =new Organizer(); + organizer.organize(page,resolution,result); + + // Check rendering + assertRendered(result,"SourceChoiceResult.xml",new PageTemplateSet(),true); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSources.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSources.xml new file mode 100644 index 00000000000..36cace66ff7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSources.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="TwoSectionsFourSources" layout="twoColumns"> + <section source="source3 source1" order="[source]" max="8" region="left"/> + <section source="source4 source2" order="-[rank]" max="10" region="right"/> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesResult.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesResult.xml new file mode 100644 index 00000000000..0ca5bfc7ea0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesResult.xml @@ -0,0 +1,65 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<result version="1.0" layout="twoColumns"> + + <section region="left"> + <hit relevance="1.0" source="source3"> + <id>source3-1</id> + </hit> + <hit relevance="0.5" source="source3"> + <id>source3-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source3"> + <id>source3-3</id> + </hit> + <hit relevance="0.25" source="source3"> + <id>source3-4</id> + </hit> + <hit relevance="0.2" source="source3"> + <id>source3-5</id> + </hit> + <hit relevance="1.0" source="source1"> + <id>source1-1</id> + </hit> + <hit relevance="0.5" source="source1"> + <id>source1-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source1"> + <id>source1-3</id> + </hit> + </section> + + <section region="right"> + <hit relevance="1.0" source="source4"> + <id>source4-1</id> + </hit> + <hit relevance="1.0" source="source2"> + <id>source2-1</id> + </hit> + <hit relevance="0.5" source="source4"> + <id>source4-2</id> + </hit> + <hit relevance="0.5" source="source2"> + <id>source2-2</id> + </hit> + <hit relevance="0.3333333333333333" source="source4"> + <id>source4-3</id> + </hit> + <hit relevance="0.3333333333333333" source="source2"> + <id>source2-3</id> + </hit> + <hit relevance="0.25" source="source4"> + <id>source4-4</id> + </hit> + <hit relevance="0.25" source="source2"> + <id>source2-4</id> + </hit> + <hit relevance="0.2" source="source4"> + <id>source4-5</id> + </hit> + <hit relevance="0.1666666666666667" source="source4"> + <id>source4-6</id> + </hit> + </section> + +</result> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesTestCase.java new file mode 100644 index 00000000000..3fff2103332 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/TwoSectionsFourSourcesTestCase.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.engine.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.engine.Organizer; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class TwoSectionsFourSourcesTestCase extends ExecutionAbstractTestCase { + + @Test + public void testExecution() { + // Create the page template + Choice page=Choice.createSingleton(importPage("TwoSectionsFourSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source2",4)); + result.hits().add(createHits("source3",12)); + result.hits().add(createHits("source4",13)); + + new Organizer().organize(page,new DeterministicResolver().resolve(page,query,result),result); + + // Check execution: + // Two subsections with two sources each, the first grouped the second blended + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertGroupedSource3Source1(section1.asList()); + assertBlendedSource4Source2(section2.asList()); + + // Check rendering + assertRendered(result,"TwoSectionsFourSourcesResult.xml"); + } + + @Test + public void testExecutionMissingOneSource() { + // Create the page template + Choice page=Choice.createSingleton(importPage("TwoSectionsFourSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source3",12)); + result.hits().add(createHits("source4",13)); + + new Organizer().organize(page,new DeterministicResolver().resolve(page,query,result),result); + + // Check execution: + // Two subsections with two sources each, the first grouped the second blended + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertGroupedSource3Source1(section1.asList()); + assertEqualHitGroups(createHits("source4",10),section2); + } + + @Test + public void testExecutionMissingTwoSources() { + // Create the page template + Choice page=Choice.createSingleton(importPage("TwoSectionsFourSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + result.hits().add(createHits("source1",3)); + result.hits().add(createHits("source3",12)); + + new Organizer().organize(page,new DeterministicResolver().resolve(page,query,result),result); + + // Check execution: + // Two subsections with two sources each, the first grouped the second blended + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertGroupedSource3Source1(section1.asList()); + assertEquals(0,section2.size()); + } + + @Test + public void testExecutionMissingAllSources() { + // Create the page template + Choice page=Choice.createSingleton(importPage("TwoSectionsFourSources.xml")); + + // Create a federated result + Query query=new Query(); + Result result=new Result(query); + + new Organizer().organize(page,new DeterministicResolver().resolve(page,query,result),result); + + // Check execution: + // Two subsections with two sources each, the first grouped the second blended + assertEquals(2,result.hits().size()); + HitGroup section1=(HitGroup)result.hits().get(0); + HitGroup section2=(HitGroup)result.hits().get(1); + assertEquals(0,section1.size()); + assertEquals(0,section2.size()); + } + + private void assertGroupedSource3Source1(List<Hit> hits) { + assertEquals(8,hits.size()); + assertEquals("source3-1",hits.get(0).getId().stringValue()); + assertEquals("source3-2",hits.get(1).getId().stringValue()); + assertEquals("source3-3",hits.get(2).getId().stringValue()); + assertEquals("source3-4",hits.get(3).getId().stringValue()); + assertEquals("source3-5",hits.get(4).getId().stringValue()); + assertEquals("source1-1",hits.get(5).getId().stringValue()); + assertEquals("source1-2",hits.get(6).getId().stringValue()); + assertEquals("source1-3",hits.get(7).getId().stringValue()); + } + + private void assertBlendedSource4Source2(List<Hit> hits) { + assertEquals(10,hits.size()); + assertEquals("source4-1",hits.get(0).getId().stringValue()); + assertEquals("source2-1",hits.get(1).getId().stringValue()); + assertEquals("source4-2",hits.get(2).getId().stringValue()); + assertEquals("source2-2",hits.get(3).getId().stringValue()); + assertEquals("source4-3",hits.get(4).getId().stringValue()); + assertEquals("source2-3",hits.get(5).getId().stringValue()); + assertEquals("source4-4",hits.get(6).getId().stringValue()); + assertEquals("source2-4",hits.get(7).getId().stringValue()); + assertEquals("source4-5",hits.get(8).getId().stringValue()); + assertEquals("source4-6",hits.get(9).getId().stringValue()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java new file mode 100644 index 00000000000..22c269e761f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/PageTemplateSearcherTestCase.java @@ -0,0 +1,220 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.intent.model.*; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.PageTemplateRegistry; +import com.yahoo.search.pagetemplates.PageTemplateSearcher; +import com.yahoo.search.pagetemplates.engine.Resolution; +import com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver; +import com.yahoo.search.pagetemplates.model.Choice; +import com.yahoo.search.pagetemplates.model.PageElement; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.text.interpretation.Interpretation; + +import java.util.*; + +/** + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class PageTemplateSearcherTestCase extends junit.framework.TestCase { + + public void testSearcher() { + PageTemplateSearcher s = new PageTemplateSearcher(createPageTemplateRegistry(), new FirstChoiceResolver()); + Chain<Searcher> chain = new Chain<>(s,new MockFederator()); + + { + // No template specified, should use default + Result result=new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=foo&page.resolver=native.deterministic")); + assertSources("source1 source2",result); + } + + { + Result result=new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=foo&page.id=oneSource&page.resolver=native.deterministic")); + assertSources("source1",result); + } + + { + Result result=new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=foo&page.id=twoSources&model.sources=source1&page.resolver=native.deterministic")); + assertSources("source1",result); + } + + { + Query query=new Query("?query=foo&page.resolver=native.deterministic"); + addIntentModelSpecifyingSource3(query); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2",result); + } + + { + Query query=new Query("?query=foo&page.id=twoSourcesAndAny&page.resolver=native.deterministic"); + addIntentModelSpecifyingSource3(query); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2 source3",result); + } + + { + Query query=new Query("?query=foo&page.id=anySource&page.resolver=native.deterministic"); + addIntentModelSpecifyingSource3(query); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source3",result); + } + + { + Query query=new Query("?query=foo&page.id=anySource&page.resolver=native.deterministic"); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertTrue(query.getModel().getSources().isEmpty()); + assertNotNull(result.hits().get("source1")); + assertNotNull(result.hits().get("source2")); + assertNotNull(result.hits().get("source3")); + } + + { + Query query=new Query("?query=foo&page.id=choiceOfSources&page.resolver=native.deterministic"); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2","source2",result); + } + + { + Query query=new Query("?query=foo&page.id=choiceOfSources&page.resolver=test.firstChoice"); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2","source1",result); + } + + { // Specifying two templates, should pick the last + Query query=new Query("?query=foo&page.id=threeSources+oneSource&page.resolver=native.deterministic"); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2 source3","source1",result); + } + + { // Specifying two templates as a list, should override the page.id setting + Query query=new Query("?query=foo&page.id=anySource&page.resolver=native.deterministic"); + query.properties().set("page.idList",Arrays.asList("oneSource","threeSources")); + Result result=new Execution(chain, Execution.Context.createContextStub()).search(query); + assertSources("source1 source2 source3","source1 source2 source3",result); + } + + { + try { + Query query=new Query("?query=foo&page.id=oneSource+choiceOfSources&page.resolver=noneSuch"); + new Execution(chain, Execution.Context.createContextStub()).search(query); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("No page template resolver 'noneSuch'",e.getMessage()); + } + } + + } + + private PageTemplateRegistry createPageTemplateRegistry() { + PageTemplateRegistry registry=new PageTemplateRegistry(); + + PageTemplate twoSources=new PageTemplate(new ComponentId("default")); + twoSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source1")); + twoSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source2")); + registry.register(twoSources); + + PageTemplate oneSource=new PageTemplate(new ComponentId("oneSource")); + oneSource.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source1")); + registry.register(oneSource); + + PageTemplate threeSources=new PageTemplate(new ComponentId("threeSources")); + threeSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source1")); + threeSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source2")); + threeSources.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source3")); + registry.register(threeSources); + + PageTemplate twoSourcesAndAny=new PageTemplate(new ComponentId("twoSourcesAndAny")); + twoSourcesAndAny.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source1")); + twoSourcesAndAny.getSection().elements().add(new com.yahoo.search.pagetemplates.model.Source("source2")); + twoSourcesAndAny.getSection().elements().add(com.yahoo.search.pagetemplates.model.Source.any); + registry.register(twoSourcesAndAny); + + PageTemplate anySource=new PageTemplate(new ComponentId("anySource")); + anySource.getSection().elements().add(com.yahoo.search.pagetemplates.model.Source.any); + registry.register(anySource); + + PageTemplate choiceOfSources=new PageTemplate(new ComponentId("choiceOfSources")); + List<PageElement> alternatives=new ArrayList<>(); + alternatives.add(new com.yahoo.search.pagetemplates.model.Source("source1")); + alternatives.add(new com.yahoo.search.pagetemplates.model.Source("source2")); + choiceOfSources.getSection().elements().add(Choice.createSingletons(alternatives)); + registry.register(choiceOfSources); + + registry.freeze(); + return registry; + } + + private void addIntentModelSpecifyingSource3(Query query) { + IntentModel intentModel=new IntentModel(); + InterpretationNode interpretation=new InterpretationNode(new Interpretation("ignored")); + IntentNode intent=new IntentNode(new Intent("ignored"),1.0); + intent.children().add(new SourceNode(new com.yahoo.search.intent.model.Source("source3"),1.0)); + interpretation.children().add(intent); + intentModel.children().add(interpretation); + intentModel.setTo(query); + } + + private void assertSources(String expectedSourceString,Result result) { + assertSources(expectedSourceString,expectedSourceString,result); + } + + private void assertSources(String expectedQuerySourceString,String expectedResultSourceString,Result result) { + Set<String> expectedQuerySources=new HashSet<>(Arrays.asList(expectedQuerySourceString.split(" "))); + assertEquals(expectedQuerySources,result.getQuery().getModel().getSources()); + + Set<String> expectedResultSources=new HashSet<>(Arrays.asList(expectedResultSourceString.split(" "))); + for (String sourceName : Arrays.asList("source1 source2 source3".split(" "))) { + if (expectedResultSources.contains(sourceName)) + assertNotNull("Result contains '" + sourceName + "'",result.hits().get(sourceName)); + else + assertNull("Result does not contain '" + sourceName + "'",result.hits().get(sourceName)); + } + } + + private static class MockFederator extends Searcher { + + @Override + public Result search(Query query,Execution execution) { + Result result=new Result(query); + for (String sourceName : Arrays.asList("source1 source2 source3".split(" "))) + if (query.getModel().getSources().isEmpty() || query.getModel().getSources().contains(sourceName)) + result.hits().add(createSource(sourceName)); + return result; + } + + private HitGroup createSource(String sourceName) { + HitGroup hitGroup=new HitGroup("source:" + sourceName); + Hit hit=new Hit(sourceName); + hit.setSource(sourceName); + hitGroup.add(hit); + return hitGroup; + } + + } + + /** Like the deterministic resolver except that it takes the <i>first</i> option of all choices */ + private static class FirstChoiceResolver extends DeterministicResolver { + + public FirstChoiceResolver() { + super("test.firstChoice"); + } + + /** Chooses the first alternative of any choice */ + @Override + public void resolve(Choice choice, Query query, Result result, Resolution resolution) { + resolution.addChoiceResolution(choice,0); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParameters.xml b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParameters.xml new file mode 100644 index 00000000000..2a98ef6918f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParameters.xml @@ -0,0 +1,16 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<page id="SourceParameters"> + <source name="source1"> + <parameter name="p1">source1p1Value</parameter> + <parameter name="p2">source1p2Value</parameter> + </source> + <choice> + <source name="source2"> + <parameter name="p1">source2p1Value</parameter> + <parameter name="p3">source2p3Value</parameter> + </source> + <source name="source3"> + <parameter name="p1">source3p1Value</parameter> + </source> + </choice> +</page> diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParametersTestCase.java new file mode 100644 index 00000000000..1f79637119a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/test/SourceParametersTestCase.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.pagetemplates.test; + +import com.yahoo.search.Query; +import com.yahoo.search.pagetemplates.PageTemplate; +import com.yahoo.search.pagetemplates.PageTemplateRegistry; +import com.yahoo.search.pagetemplates.PageTemplateSearcher; +import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class SourceParametersTestCase extends junit.framework.TestCase { + + private static final String root="src/test/java/com/yahoo/search/pagetemplates/test/"; + + public void testSourceParametersWithSourcesDeterminedByTemplate() { + // Create the page template + PageTemplateRegistry pageTemplateRegistry=new PageTemplateRegistry(); + PageTemplate page=importPage("SourceParameters.xml"); + pageTemplateRegistry.register(page); + PageTemplateSearcher s=new PageTemplateSearcher(pageTemplateRegistry); + Query query=new Query("?query=foo&page.id=SourceParameters&page.resolver=native.deterministic"); + new Execution(s, Execution.Context.createContextStub()).search(query); + assertEquals("source1p1Value",query.properties().get("source.source1.p1")); + assertEquals("source1p1Value",query.properties().get("source.source1.p1")); + assertEquals("source2p1Value",query.properties().get("source.source2.p1")); + assertEquals("source2p3Value",query.properties().get("source.source2.p3")); + assertEquals("source3p1Value",query.properties().get("source.source3.p1")); + assertEquals("We get the correct number of parameters",5,query.properties().listProperties("source").size()); + } + + public void testSourceParametersWithSourcesDeterminedByParameter() { + // Create the page template + PageTemplateRegistry pageTemplateRegistry=new PageTemplateRegistry(); + PageTemplate page=importPage("SourceParameters.xml"); + pageTemplateRegistry.register(page); + PageTemplateSearcher s=new PageTemplateSearcher(pageTemplateRegistry); + Query query=new Query("?query=foo&page.id=SourceParameters&model.sources=source1,source3&page.resolver=native.deterministic"); + new Execution(s, Execution.Context.createContextStub()).search(query); + assertEquals("source1p1Value",query.properties().get("source.source1.p1")); + assertEquals("source1p1Value",query.properties().get("source.source1.p1")); + assertEquals("source3p1Value",query.properties().get("source.source3.p1")); + assertEquals("We get the correct number of parameters",3,query.properties().listProperties("source").size()); + } + + protected PageTemplate importPage(String name) { + PageTemplate template=new PageTemplateXMLReader().readFile(root + name); + assertNotNull("Could look up read template '" + name + "'",template); + return template; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java new file mode 100644 index 00000000000..e7a0c78aa88 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + +import com.ibm.icu.lang.UScript; +import com.ibm.icu.text.Collator; +import com.ibm.icu.text.RuleBasedCollator; +import com.ibm.icu.util.ULocale; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; + +/** + * @author balder + */ +public class SortingTestCase { + @Test + public void validAttributeName() { + assertNotNull(Sorting.fromString("a")); + assertNotNull(Sorting.fromString("_a")); + assertNotNull(Sorting.fromString("+a")); + assertNotNull(Sorting.fromString("-a")); + assertNotNull(Sorting.fromString("a.b")); + try { + assertNotNull(Sorting.fromString("-1")); + fail("'-1' should not be allowed as attribute name."); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "Illegal attribute name '1' for sorting. Requires '[\\[]*[a-zA-Z_][\\.a-zA-Z0-9_-]*[\\]]*'"); + } catch (Exception e) { + fail("I only expect 'IllegalArgumentException', not: + " + e.toString()); + } + } + @Test + public void requireThatChineseSortCorrect() { + requireThatChineseHasCorrectRules(Collator.getInstance(new ULocale("zh"))); + Sorting ch = Sorting.fromString("uca(a,zh)"); + assertEquals(1, ch.fieldOrders().size()); + Sorting.FieldOrder fo = ch.fieldOrders().get(0); + assertTrue(fo.getSorter() instanceof Sorting.UcaSorter); + Sorting.UcaSorter uca = (Sorting.UcaSorter) fo.getSorter(); + requireThatChineseHasCorrectRules(uca.getCollator()); + Sorting.AttributeSorter sorter = fo.getSorter(); + assertTrue(sorter.compare("a", "b") < 0); + assertTrue(sorter.compare("a", "a\u81EA") < 0); + assertTrue(sorter.compare("\u81EA", "a") < 0); + } + + private void requireThatArabicHasCorrectRules(Collator col) { + final int reorderCodes [] = {UScript.ARABIC}; + assertEquals("6.2.0.0", col.getUCAVersion().toString()); + assertEquals("58.0.0.6", col.getVersion().toString()); + assertEquals(Arrays.toString(reorderCodes), Arrays.toString(col.getReorderCodes())); + assertTrue(col.compare("a", "b") < 0); + assertTrue(col.compare("a", "aس") < 0); + assertFalse(col.compare("س", "a") < 0); + + assertEquals(" [reorder Arab]&ت<<ة<<<ﺔ<<<ﺓ&ي<<ى<<<ﯨ<<<ﯩ<<<ﻰ<<<ﻯ<<<ﲐ<<<ﱝ", ((RuleBasedCollator) col).getRules()); + assertFalse(col.compare("س", "a") < 0); + } + + private void requireThatChineseHasCorrectRules(Collator col) { + final int reorderCodes [] = {UScript.HAN}; + assertEquals("8.0.0.0", col.getUCAVersion().toString()); + assertEquals("153.64.29.0", col.getVersion().toString()); + assertEquals(Arrays.toString(reorderCodes), Arrays.toString(col.getReorderCodes())); + + assertNotEquals("", ((RuleBasedCollator) col).getRules()); + } + @Test + @Ignore + public void requireThatArabicSortCorrect() { + requireThatArabicHasCorrectRules(Collator.getInstance(new ULocale("ar"))); + Sorting ar = Sorting.fromString("uca(a,ar)"); + assertEquals(1, ar.fieldOrders().size()); + Sorting.FieldOrder fo = ar.fieldOrders().get(0); + assertTrue(fo.getSorter() instanceof Sorting.UcaSorter); + Sorting.UcaSorter uca = (Sorting.UcaSorter) fo.getSorter(); + requireThatArabicHasCorrectRules(uca.getCollator()); + Sorting.AttributeSorter sorter = fo.getSorter(); + assertTrue(sorter.compare("a", "b") < 0); + assertTrue(sorter.compare("a", "aس") < 0); + assertTrue(sorter.compare("س", "a") < 0); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/context/test/ConcurrentTraceTestCase.java b/container-search/src/test/java/com/yahoo/search/query/context/test/ConcurrentTraceTestCase.java new file mode 100644 index 00000000000..98ed684af17 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/context/test/ConcurrentTraceTestCase.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.context.test; + +import java.util.ArrayList; +import java.util.List; + +import com.yahoo.component.chain.Chain; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.AsyncExecution; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.FutureResult; + +/** + * Checks it's OK adding more traces to an instance which is being rendered. + * + * @author <a href="arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + */ +@SuppressWarnings("deprecation") +public class ConcurrentTraceTestCase { + class TraceSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + for(int i = 0;i<1000;i++) { + query.trace("Trace", false, 1); + try { + Thread.sleep(1); + } catch (InterruptedException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + } + return execution.search(query); + } + } + + class AsyncSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Chain<Searcher> chain = new Chain<>(new TraceSearcher()); + + Result result = new Result(query); + List<FutureResult> futures = new ArrayList<>(); + for(int i = 0; i < 100; i++) { + futures.add(new AsyncExecution(chain, execution).searchAndFill(query)); + } + AsyncExecution.waitForAll(futures, 10); + return result; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/context/test/LoggingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/context/test/LoggingTestCase.java new file mode 100644 index 00000000000..bbddae0f7f0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/context/test/LoggingTestCase.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.context.test; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import com.yahoo.processing.execution.Execution; +import com.yahoo.search.Query; +import com.yahoo.search.query.context.QueryContext; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class LoggingTestCase extends junit.framework.TestCase { + + public void testLogging() { + Query query=new Query(); + QueryContext queryContext = query.getContext(true); + queryContext.logValue("a","a1"); + queryContext.trace("first message", 2); + queryContext.logValue("a","a2"); + queryContext.logValue("b","b1"); + QueryContext h2 = query.clone().getContext(true); + h2.logValue("b","b2"); + h2.trace("second message", 2); + h2.logValue("b","b3"); + queryContext.logValue("b","b4"); + QueryContext h3 = query.clone().getContext(true); + h3.logValue("b","b5"); + h3.logValue("c","c1"); + h3.trace("third message", 2); + h2.logValue("c","c2"); + queryContext.trace("fourth message", 2); + queryContext.logValue("d","d1"); + h2.trace("fifth message", 2); + h2.logValue("c","c3"); + queryContext.logValue("c","c4"); + + // Assert that all of the above is in the log, in some undefined order + Set<String> logValues=new HashSet<>(); + for (Iterator<Execution.Trace.LogValue> logValueIterator=queryContext.logValueIterator(); logValueIterator.hasNext(); ) + logValues.add(logValueIterator.next().toString()); + assertEquals(12,logValues.size()); + assertTrue(logValues.contains("a=a1")); + assertTrue(logValues.contains("a=a2")); + assertTrue(logValues.contains("b=b1")); + assertTrue(logValues.contains("b=b2")); + assertTrue(logValues.contains("b=b3")); + assertTrue(logValues.contains("b=b4")); + assertTrue(logValues.contains("b=b5")); + assertTrue(logValues.contains("c=c1")); + assertTrue(logValues.contains("c=c2")); + assertTrue(logValues.contains("d=d1")); + assertTrue(logValues.contains("c=c3")); + assertTrue(logValues.contains("c=c4")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/context/test/PropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/context/test/PropertiesTestCase.java new file mode 100644 index 00000000000..e9bdb6f60f5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/context/test/PropertiesTestCase.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.context.test; + +import com.yahoo.search.Query; +import com.yahoo.search.query.context.QueryContext; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class PropertiesTestCase extends junit.framework.TestCase { + + public void testProperties() { + Query query=new Query(); + QueryContext h = query.getContext(true); + h.setProperty("a","a1"); + h.trace("first message", 2); + h.setProperty("a","a2"); + h.setProperty("b","b1"); + query.clone(); + QueryContext h2 = query.clone().getContext(true); + h2.setProperty("b","b2"); + h2.trace("second message", 2); + h2.setProperty("b","b3"); + h.setProperty("b","b4"); + QueryContext h3 = query.clone().getContext(true); + h3.setProperty("b","b5"); + h3.setProperty("c","c1"); + h3.trace("third message", 2); + h2.setProperty("c","c2"); + h.trace("fourth message", 2); + h.setProperty("d","d1"); + h2.trace("fifth message", 2); + h2.setProperty("c","c3"); + h.setProperty("c","c4"); + + assertEquals("a2",h.getProperty("a")); + assertEquals("b5",h.getProperty("b")); + assertEquals("c4",h.getProperty("c")); + assertEquals("d1",h.getProperty("d")); + assertNull(h.getProperty("e")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/context/test/TraceTestCase.java b/container-search/src/test/java/com/yahoo/search/query/context/test/TraceTestCase.java new file mode 100644 index 00000000000..7cc3d939b01 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/context/test/TraceTestCase.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.context.test; + +import com.yahoo.search.Query; +import com.yahoo.search.query.context.QueryContext; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.Iterator; + +/** + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class TraceTestCase extends junit.framework.TestCase { + + public void testBasicTracing() { + Query query=new Query(); + QueryContext h = query.getContext(true); + h.trace("first message", 0); + h.trace("second message", 0); + assertEquals("trace: [ [ first message second message ] ]", h.toString()); + } + + public void testCloning() throws IOException { + Query query=new Query(); + QueryContext h = query.getContext(true); + h.trace("first message", 0); + QueryContext h2 = query.clone().getContext(true); + h2.trace("second message", 0); + QueryContext h3 = query.clone().getContext(true); + h3.trace("third message", 0); + h.trace("fourth message", 0); + h2.trace("fifth message", 0); + Writer w = new StringWriter(); + Writer w2 = new StringWriter(); + h2.render(w2); + String finishedBelow = w2.toString(); + h.render(w); + String toplevel = w.toString(); + // check no matter which QueryContext ends up in the final Result, + // all context info is rendered + assertEquals(toplevel, finishedBelow); + // basic sanity test + assertEquals("trace: [ [ " + + "first message second message third message " + + "fourth message fifth message ] ]",h.toString()); + Iterator<String> i = h.getTrace().traceNode().root().descendants(String.class).iterator(); + assertEquals("first message",i.next()); + assertEquals("second message",i.next()); + assertEquals("third message",i.next()); + assertEquals("fourth message",i.next()); + assertEquals("fifth message",i.next()); + } + + public void testEmpty() throws IOException { + Query query=new Query(); + QueryContext h = query.getContext(true); + Writer w = new StringWriter(); + h.render(w); + assertEquals("", w.toString()); + } + + public void testEmptySubSequence() { + Query query=new Query(); + QueryContext h = query.getContext(true); + query.clone().getContext(true); + Writer w = new StringWriter(); + try { + h.render(w); + } catch (IOException e) { + assertTrue("rendering empty subsequence crashed", false); + } + } + + public void testAttachedTraces() throws IOException { + String needle0 = "nalle"; + String needle1 = "tralle"; + String needle2 = "trolle"; + String needle3 = "bamse"; + Query q = new Query("/?tracelevel=1"); + q.trace(needle0, false, 1); + Query q2 = new Query(); + q.attachContext(q2); + q2.trace(needle1, false, 1); + q2.trace(needle2, false, 1); + q.trace(needle3, false, 1); + Writer w = new StringWriter(); + q.getContext(false).render(w); + String trace = w.toString(); + int nalle = trace.indexOf(needle0); + int tralle = trace.indexOf(needle1); + int trolle = trace.indexOf(needle2); + int bamse = trace.indexOf(needle3); + assertTrue("Could not find first manual context to main query.", nalle > 0); + assertTrue("Could not find second manual context to main query.", bamse > 0); + assertTrue("Could not find first manual context to attached query.", tralle > 0); + assertTrue("Could not find second manual context to attached query.", trolle > 0); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/MultiProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/MultiProfileTestCase.java new file mode 100644 index 00000000000..f406552cd30 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/MultiProfileTestCase.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.config.test; + +import java.util.HashMap; +import java.util.Map; + +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; + +/** + * @author bratseth + */ +public class MultiProfileTestCase extends junit.framework.TestCase { + + public void testValid() { + QueryProfileRegistry registry= + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/multiprofile"); + + QueryProfile multiprofile1=registry.getComponent("multiprofile1"); + assertNotNull(multiprofile1); + assertGet("general-a","a",new String[] {null,null,null},multiprofile1); + assertGet("us-nokia-test1-a","a",new String[] {"us","nok ia","test1"},multiprofile1); + assertGet("us-nokia-b","b",new String[] {"us","nok ia","test1"},multiprofile1); + assertGet("us-a","a",new String[] {"us",null,null},multiprofile1); + assertGet("us-b","b",new String[] {"us",null,null},multiprofile1); + assertGet("us-nokia-a","a",new String[] {"us","nok ia",null},multiprofile1); + assertGet("us-test1-a","a",new String[] {"us",null,"test1"},multiprofile1); + assertGet("us-test1-b","b",new String[] {"us",null,"test1"},multiprofile1); + + assertGet("us-a","a",new String[] {"us","unspecified","unspecified"},multiprofile1); + assertGet("us-nokia-a","a",new String[] {"us","nok ia","unspecified"},multiprofile1); + assertGet("us-test1-a","a",new String[] {"us","unspecified","test1"},multiprofile1); + assertGet("us-nokia-b","b",new String[] {"us","nok ia","test1"},multiprofile1); + + // ...inherited + assertGet("parent1-value","parent1",new String[] { "us","nok ia","-" }, multiprofile1); + assertGet("parent2-value","parent2",new String[] { "us","nok ia","-" }, multiprofile1); + assertGet(null,"parent1",new String[] { "us","-","-" }, multiprofile1); + assertGet(null,"parent2",new String[] { "us","-","-" }, multiprofile1); + } + + private void assertGet(String expectedValue,String parameter,String[] dimensionValues,QueryProfile profile) { + Map<String,String> context=new HashMap<>(); + context.put("region",dimensionValues[0]); + context.put("model",dimensionValues[1]); + context.put("bucket",dimensionValues[2]); + assertEquals("Looking up '" + parameter + "' for '" + toString(dimensionValues) + "'",expectedValue,profile.get(parameter,context,null)); + } + + private String toString(String[] array) { + StringBuilder b=new StringBuilder("["); + for (String value : array) { + b.append(value); + b.append(","); + } + b.deleteCharAt(b.length()-1); // Remove last comma :-) + b.append("]"); + return b.toString(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java new file mode 100644 index 00000000000..36fc16b94eb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileConfigurationTestCase.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.config.test; + +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.io.IOUtils; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.test.QueryTestCase; +import com.yahoo.vespa.config.ConfigPayload; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.*; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * @author bratseth + */ +public class QueryProfileConfigurationTestCase { + + public final String CONFIG_DIR ="src/test/java/com/yahoo/search/query/profile/config/test/"; + + @Test + public void testConfiguration() { + QueryProfileConfigurer configurer= + new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profiles-configuration.cfg"); + QueryProfile profile=configurer.getCurrentRegistry().getComponent("default"); + + assertEquals("a-value",profile.get("a")); + assertEquals("b-value",profile.get("b")); + assertEquals("c.d-value",profile.get("c.d")); + assertFalse(profile.isDeclaredOverridable("c.d", null)); + assertEquals("e-value-inherited1",profile.get("e")); + assertEquals("g.d2-value-inherited1",profile.get("g.d2")); // Even though we make an explicit reference to one not having this value, we still inherit it + assertEquals("a-value-subprofile1",profile.get("sub1.a")); + assertEquals("c.d-value-subprofile1",profile.get("sub1.c.d")); + assertEquals("a-value-subprofile2",profile.get("sub2.a")); + assertEquals("c.d-value-subprofile2",profile.get("sub2.c.d")); + assertEquals("e-value-subprofile3",profile.get("g.e")); + } + + @Test + public void testBug3197426() { + QueryProfileConfigurer configurer = new QueryProfileConfigurer("file:" + CONFIG_DIR + "bug3197426.cfg"); + CompiledQueryProfile profile = configurer.getCurrentRegistry().getComponent("default").compile(null); + Map<String, Object> properties = new QueryProfileProperties(profile).listProperties("source.image"); + assertEquals("yes", properties.get("mlr")); + assertEquals("zh-Hant", properties.get("language")); + assertEquals("tw", properties.get("custid2")); + assertEquals("4", properties.get("hits")); + assertEquals("0", properties.get("offset")); + assertEquals("image", properties.get("catalog")); + assertEquals("yahoo", properties.get("custid1")); + assertEquals("utf-8", properties.get("encoding")); + assertEquals("all", properties.get("imquality")); + assertEquals("all", properties.get("dimensions")); + assertEquals("1", properties.get("flickr")); + assertEquals("yes", properties.get("ocr")); + } + + @Test + public void testVariantConfiguration() { + QueryProfileConfigurer configurer= + new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants-configuration.cfg"); + + // Variant 1 + QueryProfile variants1 =configurer.getCurrentRegistry().getComponent("variants1"); + assertGet("x1.y1.a","a",new String[] { "x1","y1" }, variants1); + assertGet("x1.y1.b","b",new String[] { "x1","y1" }, variants1); + assertGet("x1.y?.a","a",new String[] { "x1","zz" }, variants1); + assertGet("x?.y1.a","a",new String[] { "zz","y1" }, variants1); + assertGet("a-deflt","a",new String[] { "z1","z2" }, variants1); + // ...inherited + assertGet("parent1-value","parent1",new String[] { "x1","y1" }, variants1); + assertGet("parent2-value","parent2",new String[] { "x1","y1" }, variants1); + assertGet(null,"parent1",new String[] { "x1","y2" }, variants1); + assertGet(null,"parent2",new String[] { "x1","y2" }, variants1); + + // Variant 2 + QueryProfile variants2 =configurer.getCurrentRegistry().getComponent("variants2"); + assertGet("variant2:y1.c","c",new String[] { "*","y1" }, variants2); + assertGet("variant2:y2.c","c",new String[] { "*","y2" }, variants2); + assertGet("variant2:c-df","c",new String[] { "*","z1" }, variants2); + assertGet("variant2:c-df","c",new String[] { }, variants2); + assertGet("variant2:c-df","c",new String[] { "*" }, variants2); + assertGet(null, "d",new String[] { "*","y1" }, variants2); + + // Reference following from variant 1 + assertGet("variant2:y1.c","toVariants.c",new String[] { "**", "y1" } , variants1); + assertGet("variant3:c-df","toVariants.c",new String[] { "x1", "**" } , variants1); + assertGet("variant3:y1.c","toVariants.c",new String[] { "x1", "y1" } , variants1); // variant3 by order priority + assertGet("variant3:y2.c","toVariants.c",new String[] { "x1", "y2" } , variants1); + } + + @Test + public void testVariantConfigurationThroughQueryLookup() { + QueryProfileConfigurer configurer= + new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants-configuration.cfg"); + + CompiledQueryProfileRegistry registry = configurer.getCurrentRegistry().compile(); + CompiledQueryProfile variants1 = registry.getComponent("variants1"); + + // Variant 1 + assertEquals("x1.y1.a", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y1"), variants1).properties().get("a")); + assertEquals("x1.y1.b", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y1"), variants1).properties().get("b")); + assertEquals("x1.y1.defaultIndex", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y1"), variants1).getModel().getDefaultIndex()); + assertEquals("x1.y?.a", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=zz"), variants1).properties().get("a")); + assertEquals("x1.y?.defaultIndex", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=zz"),variants1).getModel().getDefaultIndex()); + assertEquals("x?.y1.a", new Query(QueryTestCase.httpEncode("?query=foo&x=zz&y=y1"), variants1).properties().get("a")); + assertEquals("x?.y1.defaultIndex", new Query(QueryTestCase.httpEncode("?query=foo&x=zz&y=y1"), variants1).getModel().getDefaultIndex()); + assertEquals("x?.y1.filter", new Query(QueryTestCase.httpEncode("?query=foo&x=zz&y=y1"), variants1).getModel().getFilter()); + assertEquals("a-deflt", new Query(QueryTestCase.httpEncode("?query=foo&x=z1&y=z2"), variants1).properties().get("a")); + + // Variant 2 + CompiledQueryProfile variants2 = registry.getComponent("variants2"); + assertEquals("variant2:y1.c", new Query(QueryTestCase.httpEncode("?query=foo&x=*&y=y1"), variants2).properties().get("c")); + assertEquals("variant2:y2.c", new Query(QueryTestCase.httpEncode("?query=foo&x=*&y=y2"), variants2).properties().get("c")); + assertEquals("variant2:c-df", new Query(QueryTestCase.httpEncode("?query=foo&x=*&y=z1"), variants2).properties().get("c")); + assertEquals("variant2:c-df", new Query(QueryTestCase.httpEncode("?query=foo"), variants2).properties().get("c")); + assertEquals("variant2:c-df", new Query(QueryTestCase.httpEncode("?query=foo&x=x1"), variants2).properties().get("c")); + assertNull(new Query(QueryTestCase.httpEncode("?query=foo&x=*&y=y1"), variants2).properties().get("d")); + + // Reference following from variant 1 + assertEquals("variant2:y1.c", new Query(QueryTestCase.httpEncode("?query=foo&x=**&y=y1"), variants1).properties().get("toVariants.c")); + assertEquals("variant3:c-df", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=**"), variants1).properties().get("toVariants.c")); + assertEquals("variant3:y1.c", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y1"), variants1).properties().get("toVariants.c")); + assertEquals("variant3:y2.c", new Query(QueryTestCase.httpEncode("?query=foo&x=x1&y=y2"), variants1).properties().get("toVariants.c")); + } + + @Test + public void testVariant2ConfigurationThroughQueryLookup() { + QueryProfileConfigurer configurer= + new QueryProfileConfigurer("file:" + CONFIG_DIR + "query-profile-variants2.cfg"); + + CompiledQueryProfileRegistry registry = configurer.getCurrentRegistry().compile(); + Query query = new Query(QueryTestCase.httpEncode("?query=heh&queryProfile=multi&myindex=default&myquery=lo ve&tracelevel=5"), + registry.findQueryProfile("multi")); + assertEquals("love",query.properties().get("model.queryString")); + assertEquals("default",query.properties().get("model.defaultIndex")); + + assertEquals("-20",query.properties().get("ranking.features.query(scorelimit)")); + assertEquals("-20",query.getRanking().getFeatures().get("query(scorelimit)")); + query.properties().set("rankfeature.query(scorelimit)", -30); + assertEquals("-30",query.properties().get("ranking.features.query(scorelimit)")); + assertEquals("-30",query.getRanking().getFeatures().get("query(scorelimit)")); + } + + private void assertGet(String expectedValue,String parameter,String[] dimensionValues,QueryProfile profile) { + Map<String,String> context=new HashMap<>(); + for (int i=0; i<dimensionValues.length; i++) + context.put(profile.getVariants().getDimensions().get(i),dimensionValues[i]); // Lookup dim. names to ease test... + assertEquals("Looking up '" + parameter + "' for '" + Arrays.toString(dimensionValues) + "'",expectedValue,profile.get(parameter, context, null)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java new file mode 100644 index 00000000000..11698b2b70d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/QueryProfileIntegrationTestCase.java @@ -0,0 +1,171 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.config.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.component.chain.dependencies.Provides; +import com.yahoo.container.Container; +import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.handler.HttpSearchResponse; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * Tests using query profiles in searches + * + * @author bratseth + */ +public class QueryProfileIntegrationTestCase extends junit.framework.TestCase { + + @Override + public void tearDown() { + System.getProperties().remove("config.id"); + } + + public void testUntyped() { + String configId = "dir:src/test/java/com/yahoo/search/query/profile/config/test/untyped"; + System.setProperty("config.id", configId); + Container container = new Container(); + HandlersConfigurerTestWrapper configurer = new HandlersConfigurerTestWrapper(container, configId); + SearchHandler searchHandler = (SearchHandler) configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + + // Should get "default" query profile containing the "test" search chain containing the "test" searcher + HttpRequest request = HttpRequest.createTestRequest("search", Method.GET); + HttpSearchResponse response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertNotNull(response.getResult().hits().get("from:test")); + + // Should get the "test' query profile containing the "default" search chain containing the "default" searcher + request = HttpRequest.createTestRequest("search?queryProfile=test", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertNotNull(response.getResult().hits().get("from:default")); + + // Should get "default" query profile, but override the search chain to default + request = HttpRequest.createTestRequest("search?searchChain=default", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertNotNull(response.getResult().hits().get("from:default")); + + // Tests a profile setting hits and offset + request = HttpRequest.createTestRequest("search?queryProfile=hitsoffset", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertEquals(20,response.getQuery().getHits()); + assertEquals(80,response.getQuery().getOffset()); + + // Tests a non-resolved profile request + request = HttpRequest.createTestRequest("search?queryProfile=none", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertNotNull("Got an error",response.getResult().hits().getError()); + assertEquals("Could not resolve query profile 'none'",response.getResult().hits().getError().getDetailedMessage()); + + // Tests that properties in objects owned by query is handled correctly + request = HttpRequest.createTestRequest("search?query=word&queryProfile=test", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertEquals("index",response.getQuery().getModel().getDefaultIndex()); + assertEquals("index:word",response.getQuery().getModel().getQueryTree().toString()); + configurer.shutdown(); + } + + public void testTyped() { + String configId = "dir:src/test/java/com/yahoo/search/query/profile/config/test/typed"; + System.setProperty("config.id", configId); + Container container = new Container(); + HandlersConfigurerTestWrapper configurer = new HandlersConfigurerTestWrapper(container, configId); + SearchHandler searchHandler = (SearchHandler) configurer.getRequestHandlerRegistry().getComponent(SearchHandler.class.getName()); + + // Should get "default" query profile containing the "test" search chain containing the "test" searcher + HttpRequest request = HttpRequest.createTestRequest("search", Method.GET); + HttpSearchResponse response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertNotNull(response.getResult().hits().get("from:test")); + + // Should get the "test' query profile containing the "default" search chain containing the "default" searcher + request = HttpRequest.createTestRequest("search?queryProfile=test", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertNotNull(response.getResult().hits().get("from:default")); + + // Should get "default" query profile, but override the search chain to default + request = HttpRequest.createTestRequest("search?searchChain=default", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertNotNull(response.getResult().hits().get("from:default")); + + // Tests a profile setting hits and offset + request = HttpRequest.createTestRequest("search?queryProfile=hitsoffset", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertEquals(22,response.getQuery().getHits()); + assertEquals(80,response.getQuery().getOffset()); + + // Tests a non-resolved profile request + request = HttpRequest.createTestRequest("search?queryProfile=none", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertNotNull("Got an error",response.getResult().hits().getError()); + assertEquals("Could not resolve query profile 'none'",response.getResult().hits().getError().getDetailedMessage()); + + // Test overriding a sub-profile in the request + request = HttpRequest.createTestRequest("search?queryProfile=root&sub=newsub", Method.GET); + response = (HttpSearchResponse)searchHandler.handle(request); // Cast to access content directly + assertEquals("newsubvalue1",response.getQuery().properties().get("sub.value1")); + assertEquals("newsubvalue2",response.getQuery().properties().get("sub.value2")); + configurer.shutdown(); + } + + public static class DefaultSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + result.hits().add(new Hit("from:default")); + return result; + } + + } + + public static class TestSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + result.hits().add(new Hit("from:test")); + return result; + } + + } + + /** Tests searcher communication - setting */ + @Provides("SomeObject") + public static class SettingSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + SomeObject.setTo(query,new SomeObject()); + return execution.search(query); + } + + } + + /** Tests searcher communication - receiving */ + @After("SomeObject") + public static class ReceivingSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + assertNotNull(SomeObject.getFrom(query)); + assertEquals(SomeObject.class,SomeObject.getFrom(query).getClass()); + return execution.search(query); + } + + } + + /** An example of a model object */ + private static class SomeObject { + + public static void setTo(Query query,SomeObject someObject) { + query.properties().set("SomeObject",someObject); + } + + public static SomeObject getFrom(Query query) { + // In some cases we want to create if this does not exist here + return (SomeObject)query.properties().get("SomeObject"); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/TypedProfilesConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/TypedProfilesConfigurationTestCase.java new file mode 100644 index 00000000000..b6a94396385 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/TypedProfilesConfigurationTestCase.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.config.test; + +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileConfigurer; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +/** + * @author bratseth + */ +public class TypedProfilesConfigurationTestCase extends junit.framework.TestCase { + + /** Asserts that everything is read correctly from this configuration */ + public void testIt() { + QueryProfileConfigurer configurer= + new QueryProfileConfigurer("file:src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg"); + QueryProfileRegistry registry=configurer.getCurrentRegistry(); + QueryProfileTypeRegistry types=registry.getTypeRegistry(); + + // Assert that each type was read correctly + + QueryProfileType testType=types.getComponent("testtype"); + assertEquals("testtype",testType.getId().getName()); + assertFalse(testType.isStrict()); + assertFalse(testType.getMatchAsPath()); + assertEquals(7,testType.fields().size()); + assertEquals("myString",testType.getField("myString").getName()); + assertTrue(testType.getField("myString").isMandatory()); + assertTrue(testType.getField("myString").isOverridable()); + assertFalse(testType.getField("myInteger").isMandatory()); + assertFalse(testType.getField("myInteger").isOverridable()); + FieldDescription field= testType.getField("myUserQueryProfile"); + assertEquals("reference to a query profile of type 'user'",field.getType().toInstanceDescription()); + assertTrue(field.getAliases().contains("myqp")); + assertTrue(field.getAliases().contains("user-profile")); + + QueryProfileType testTypeStrict=types.getComponent("testtypestrict"); + assertTrue(testTypeStrict.isStrict()); + assertTrue(testTypeStrict.getMatchAsPath()); + assertEquals(7,testTypeStrict.fields().size()); + assertEquals("reference to a query profile of type 'userstrict'", + testTypeStrict.getField("myUserQueryProfile").getType().toInstanceDescription()); + + QueryProfileType user=types.getComponent("user"); + assertFalse(user.isStrict()); + assertFalse(user.getMatchAsPath()); + assertEquals(2,user.fields().size()); + assertEquals(String.class,user.getField("myUserString").getType().getValueClass()); + + QueryProfileType userStrict=types.getComponent("userstrict"); + assertTrue(userStrict.isStrict()); + assertFalse(userStrict.getMatchAsPath()); + assertEquals(2,userStrict.fields().size()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java new file mode 100644 index 00000000000..3f2b437f755 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java @@ -0,0 +1,421 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.config.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.yolean.Exceptions; +import com.yahoo.search.Query; +import com.yahoo.search.query.Properties; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class XmlReadingTestCase { + + @Test + public void testValid() { + QueryProfileRegistry registry= + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/validxml"); + CompiledQueryProfileRegistry cRegistry= registry.compile(); + + QueryProfileType rootType=registry.getType("rootType"); + assertEquals(1,rootType.inherited().size()); + assertEquals("native",rootType.inherited().get(0).getId().getName()); + assertTrue(rootType.isStrict()); + assertTrue(rootType.getMatchAsPath()); + FieldDescription timeField=rootType.getField("time"); + assertTrue(timeField.isMandatory()); + assertEquals("long",timeField.getType().toInstanceDescription()); + FieldDescription userField=rootType.getField("user"); + assertFalse(userField.isMandatory()); + assertEquals("reference to a query profile of type 'user'",userField.getType().toInstanceDescription()); + + QueryProfileType user=registry.getType("user"); + assertEquals(0,user.inherited().size()); + assertFalse(user.isStrict()); + assertFalse(user.getMatchAsPath()); + assertTrue(userField.isOverridable()); + FieldDescription ageField=user.getField("age"); + assertTrue(ageField.isMandatory()); + assertEquals("integer",ageField.getType().toInstanceDescription()); + FieldDescription robotField=user.getField("robot"); + assertFalse(robotField.isMandatory()); + assertFalse(robotField.isOverridable()); + assertEquals("boolean",robotField.getType().toInstanceDescription()); + + CompiledQueryProfile defaultProfile=cRegistry.getComponent("default"); + assertNull(defaultProfile.getType()); + assertEquals("20",defaultProfile.get("hits")); + assertFalse(defaultProfile.isOverridable(new CompoundName("hits"), null)); + assertFalse(defaultProfile.isOverridable(new CompoundName("user.trusted"), null)); + assertEquals("false",defaultProfile.get("user.trusted")); + + CompiledQueryProfile referencingProfile=cRegistry.getComponent("referencingModelSettings"); + assertNull(referencingProfile.getType()); + assertEquals("some query",referencingProfile.get("model.queryString")); + assertEquals("aDefaultIndex",referencingProfile.get("model.defaultIndex")); + + // Request parameters here should be ignored + HttpRequest request=HttpRequest.createTestRequest("?query=foo&user.trusted=true&default-index=title", Method.GET); + Query query=new Query(request, defaultProfile); + assertEquals("false",query.properties().get("user.trusted")); + assertEquals("default",query.getModel().getDefaultIndex()); + assertEquals("default",query.properties().get("default-index")); + + CompiledQueryProfile rootProfile=cRegistry.getComponent("root"); + assertEquals("rootType",rootProfile.getType().getId().getName()); + assertEquals(30,rootProfile.get("hits")); + assertEquals(3,rootProfile.get("traceLevel")); + assertTrue(rootProfile.isOverridable(new CompoundName("hits"), null)); + + QueryProfile someUser=registry.getComponent("someUser"); + assertEquals("5",someUser.get("sub.test")); + assertEquals(18,someUser.get("age")); + + // aliases + assertEquals(18,someUser.get("alder")); + assertEquals(18,someUser.get("anno")); + assertEquals(18,someUser.get("aLdER")); + assertEquals(18,someUser.get("ANNO")); + assertNull(someUser.get("Age")); // Only aliases are case insensitive + + Map<String, String> context = new HashMap<>(); + context.put("x", "x1"); + assertEquals(37, someUser.get("alder", context, null)); + assertEquals(37,someUser.get("anno", context, null)); + assertEquals(37,someUser.get("aLdER", context, null)); + assertEquals(37,someUser.get("ANNO", context, null)); + assertEquals("male",someUser.get("gender", context, null)); + assertEquals("male",someUser.get("sex", context, null)); + assertEquals("male",someUser.get("Sex", context, null)); + assertNull(someUser.get("Gender", context, null)); // Only aliases are case insensitive + } + + @Test + public void testBasicsNoProfile() { + Query q=new Query(HttpRequest.createTestRequest("?query=test", Method.GET)); + assertEquals("test",q.properties().get("query")); + assertEquals("test",q.properties().get("QueRY")); + assertEquals("test",q.properties().get("model.queryString")); + assertEquals("test",q.getModel().getQueryString()); + } + + @Test + public void testBasicsWithProfile() { + QueryProfile p = new QueryProfile("default"); + p.set("a", "foo", null); + Query q=new Query(HttpRequest.createTestRequest("?query=test", Method.GET), p.compile(null)); + assertEquals("test", q.properties().get("query")); + assertEquals("test", q.properties().get("QueRY")); + assertEquals("test", q.properties().get("model.queryString")); + assertEquals("test",q.getModel().getQueryString()); + } + + /** Tests a subset of the configuration in the system test of this */ + @Test + public void testSystemtest() { + String queryString = "?query=test"; + + QueryProfileXMLReader reader = new QueryProfileXMLReader(); + CompiledQueryProfileRegistry registry = reader.read("src/test/java/com/yahoo/search/query/profile/config/test/systemtest/").compile(); + HttpRequest request = HttpRequest.createTestRequest(queryString, Method.GET); + CompiledQueryProfile profile = registry.findQueryProfile("default"); + Query query = new Query(request, profile); + Properties p = query.properties(); + + assertEquals("test", query.getModel().getQueryString()); + assertEquals("test",p.get("query")); + assertEquals("test",p.get("QueRY")); + assertEquals("test",p.get("model.queryString")); + assertEquals("bar",p.get("foo")); + assertEquals(5,p.get("hits")); + assertEquals("tit",p.get("subst")); + assertEquals("le",p.get("subst.end")); + assertEquals("title",p.get("model.defaultIndex")); + + Map<String,Object> ps = p.listProperties(); + assertEquals(6,ps.size()); + assertEquals("bar",ps.get("foo")); + assertEquals("5",ps.get("hits")); + assertEquals("tit",ps.get("subst")); + assertEquals("le",ps.get("subst.end")); + assertEquals("title",ps.get("model.defaultIndex")); + assertEquals("test",ps.get("model.queryString")); + } + + @Test + public void testInvalid1() { + try { + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/invalidxml1"); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("Error reading query profile 'illegalSetting' of type 'native': Could not set 'model.notDeclared' to 'value': 'notDeclared' is not declared in query profile type 'model', and the type is strict", Exceptions.toMessageString(e)); + } + } + + @Test + public void testInvalid2() { + try { + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/invalidxml2"); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not parse 'unparseable.xml', error at line 2, column 21: Element type \"query-profile\" must be followed by either attribute specifications, \">\" or \"/>\".", Exceptions.toMessageString(e)); + } + } + + @Test + public void testInvalid3() { + try { + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/invalidxml3"); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("The file name of query profile 'MyProfile' must be 'MyProfile.xml' but was 'default.xml'", Exceptions.toMessageString(e)); + } + } + + @Test + public void testQueryProfileVariants() { + String query = "?query=test&dim1=yahoo&dim2=uk&dim3=test"; + + QueryProfileXMLReader reader = new QueryProfileXMLReader(); + CompiledQueryProfileRegistry registry = reader.read("src/test/java/com/yahoo/search/query/profile/config/test/news/").compile(); + HttpRequest request = HttpRequest.createTestRequest(query, Method.GET); + CompiledQueryProfile profile = registry.findQueryProfile("default"); + Query q = new Query(request, profile); + + assertEquals("c", q.properties().get("a.c")); + assertEquals("b", q.properties().get("a.b")); + } + + @Test + public void testNewsFE1() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newsfe").compile(); + + String queryString="tiled?vertical=news&query=barack&intl=us&resulttypes=article&testid=&clientintl=us&SpellState=&rss=0&tracelevel=5"; + + Query query=new Query(HttpRequest.createTestRequest(queryString, Method.GET), registry.getComponent("default")); + assertEquals("13",query.properties().listProperties().get("source.news.discovery.sources.count")); + assertEquals("13",query.properties().get("source.news.discovery.sources.count")); + assertEquals("sources",query.properties().listProperties().get("source.news.discovery")); + assertEquals("sources",query.properties().get("source.news.discovery")); + } + + @Test + public void testQueryProfileVariants2() { + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2").compile(); + CompiledQueryProfile multi = registry.getComponent("multi"); + + { + Query query=new Query(HttpRequest.createTestRequest("?queryProfile=multi", Method.GET), multi); + query.validate(); + assertEquals("best",query.properties().get("model.queryString")); + assertEquals("best",query.getModel().getQueryString()); + } + { + Query query=new Query(HttpRequest.createTestRequest("?queryProfile=multi&myindex=default", Method.GET), multi); + query.validate(); + assertEquals("best", query.properties().get("model.queryString")); + assertEquals("best", query.getModel().getQueryString()); + assertEquals("default", query.getModel().getDefaultIndex()); + } + { + Query query=new Query(HttpRequest.createTestRequest("?queryProfile=multi&myindex=default&myquery=love", Method.GET), multi); + query.validate(); + assertEquals("love", query.properties().get("model.queryString")); + assertEquals("love", query.getModel().getQueryString()); + assertEquals("default", query.getModel().getDefaultIndex()); + } + { + Query query=new Query(HttpRequest.createTestRequest("?model=querybest", Method.GET), multi); + query.validate(); + assertEquals("best",query.getModel().getQueryString()); + assertEquals("title",query.properties().get("model.defaultIndex")); + assertEquals("title",query.getModel().getDefaultIndex()); + } + } + + @Test + public void testKlee() { + QueryProfileRegistry registry= + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/klee"); + + QueryProfile pv=registry.getComponent("twitter_dd-us:0.2.4"); + assertEquals("0.2.4",pv.getId().getVersion().toString()); + assertEquals("[query profile 'production']",pv.inherited().toString()); + + QueryProfile p=registry.getComponent("twitter_dd-us:0.0.0"); + assertEquals("",p.getId().getVersion().toString()); // that is 0.0.0 + assertEquals("[query profile 'twitter_dd']",p.inherited().toString()); + } + + @Test + public void testVersions() { + QueryProfileRegistry registry= + new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/versions"); + registry.freeze(); + + assertEquals("1.20.100",registry.findQueryProfile("testprofile:1.20.100").getId().getVersion().toString()); + assertEquals("1.20.100",registry.findQueryProfile("testprofile:1.20").getId().getVersion().toString()); + assertEquals("1.20.100",registry.findQueryProfile("testprofile:1").getId().getVersion().toString()); + assertEquals("1.20.100",registry.findQueryProfile("testprofile").getId().getVersion().toString()); + } + + @Test + public void testNewsFE2() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newsfe2").compile(); + + String queryString="tiled?query=a&intl=tw&mode=adv&mode=adv"; + + Query query=new Query(HttpRequest.createTestRequest(queryString, Method.GET),registry.getComponent("default")); + assertEquals("news_adv",query.properties().listProperties().get("provider")); + assertEquals("news_adv",query.properties().get("provider")); + } + + @Test + public void testSourceProvider() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider").compile(); + + String queryString="tiled?query=india&queryProfile=myprofile&source.common.intl=tw&source.common.mode=adv"; + + Query query=new Query(HttpRequest.createTestRequest(queryString, Method.GET), registry.getComponent("myprofile")); + for (Map.Entry e : query.properties().listProperties().entrySet()) + System.out.println(e); + assertEquals("news",query.properties().listProperties().get("source.common.provider")); + assertEquals("news",query.properties().get("source.common.provider")); + } + + @Test + public void testNewsCase1() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase1").compile(); + + Query query; + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); + assertEquals("0.0",query.properties().get("ranking.features.b")); + assertEquals("0.0",query.properties().listProperties().get("ranking.features.b")); + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); + assertEquals("0.1",query.properties().get("ranking.features.b")); + assertEquals("0.1",query.properties().listProperties().get("ranking.features.b")); + } + + @Test + public void testNewsCase2() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase2").compile(); + + Query query; + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); + assertEquals("0.0",query.properties().get("a.features.b")); + assertEquals("0.0",query.properties().listProperties().get("a.features.b")); + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); + assertEquals("0.1",query.properties().get("a.features.b")); + assertEquals("0.1",query.properties().listProperties().get("a.features.b")); + } + + @Test + public void testNewsCase3() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase3").compile(); + + Query query; + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); + assertEquals("0.0",query.properties().get("a.features")); + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); + assertEquals("0.1",query.properties().get("a.features")); + } + + // Should cause an exception on the first line as we are trying to create a profile setting an illegal value in "ranking" + @Test + public void testNewsCase4() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/newscase4").compile(); + + Query query; + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent", Method.GET),registry.getComponent("default")); + assertEquals("0.0",query.properties().get("ranking.features")); + query=new Query(HttpRequest.createTestRequest("?query=test&custid_1=parent&custid_2=child", Method.GET),registry.getComponent("default")); + assertEquals("0.1",query.properties().get("ranking.features")); + } + + @Test + public void testVersionRefs() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/versionrefs").compile(); + + Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); + assertEquals("MyProfile:1.0.2",query.properties().get("profile1.name")); + } + + @Test + public void testRefOverride() { + CompiledQueryProfileRegistry registry = new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverride").compile(); + + { + // Original reference + Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); + assertEquals(null,query.properties().get("profileRef")); + assertEquals("MyProfile1",query.properties().get("profileRef.name")); + assertEquals("myProfile1Only",query.properties().get("profileRef.myProfile1Only")); + assertNull(query.properties().get("profileRef.myProfile2Only")); + } + + { + // Overridden reference + Query query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); + assertEquals(null,query.properties().get("profileRef")); + assertEquals("MyProfile2",query.properties().get("profileRef.name")); + assertEquals("myProfile2Only",query.properties().get("profileRef.myProfile2Only")); + assertNull(query.properties().get("profileRef.myProfile1Only")); + + // later assignment + query.properties().set("profileRef.name","newName"); + assertEquals("newName",query.properties().get("profileRef.name")); + // ...will not impact others + query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); + assertEquals("MyProfile2",query.properties().get("profileRef.name")); + } + + } + + @Test + public void testRefOverrideTyped() { + CompiledQueryProfileRegistry registry=new QueryProfileXMLReader().read("src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped").compile(); + + { + // Original reference + Query query=new Query(HttpRequest.createTestRequest("?query=test", Method.GET),registry.getComponent("default")); + assertEquals(null,query.properties().get("profileRef")); + assertEquals("MyProfile1",query.properties().get("profileRef.name")); + assertEquals("myProfile1Only",query.properties().get("profileRef.myProfile1Only")); + assertNull(query.properties().get("profileRef.myProfile2Only")); + } + + { + // Overridden reference + Query query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=MyProfile2", Method.GET),registry.getComponent("default")); + assertEquals(null,query.properties().get("profileRef")); + assertEquals("MyProfile2",query.properties().get("profileRef.name")); + assertEquals("myProfile2Only",query.properties().get("profileRef.myProfile2Only")); + assertNull(query.properties().get("profileRef.myProfile1Only")); + + // later assignment + query.properties().set("profileRef.name","newName"); + assertEquals("newName",query.properties().get("profileRef.name")); + // ...will not impact others + query=new Query(HttpRequest.createTestRequest("?query=test&profileRef=ref:MyProfile2", Method.GET),registry.getComponent("default")); + assertEquals("MyProfile2",query.properties().get("profileRef.name")); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/bug3197426.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/bug3197426.cfg new file mode 100644 index 00000000000..03422bc1020 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/bug3197426.cfg @@ -0,0 +1,45 @@ +queryprofile[4] +queryprofile[0].id "default" +queryprofile[0].reference[1] +queryprofile[0].reference[0].name "source" +queryprofile[0].reference[0].value "source" +queryprofile[1].id "source" +queryprofile[1].reference[1] +queryprofile[1].reference[0].name "image" +queryprofile[1].reference[0].value "imageProfileTW" +queryprofile[2].id "imageProfileBase" +queryprofile[2].property[11] +queryprofile[2].property[0].name "hits" +queryprofile[2].property[0].value "4" +queryprofile[2].property[1].name "offset" +queryprofile[2].property[1].value "0" +queryprofile[2].property[2].name "catalog" +queryprofile[2].property[2].value "image" +queryprofile[2].property[3].name "custid1" +queryprofile[2].property[3].value "yahoo" +queryprofile[2].property[4].name "custid2" +queryprofile[2].property[4].value "us" +queryprofile[2].property[5].name "language" +queryprofile[2].property[5].value "en" +queryprofile[2].property[6].name "encoding" +queryprofile[2].property[6].value "utf-8" +queryprofile[2].property[7].name "imquality" +queryprofile[2].property[7].value "all" +queryprofile[2].property[8].name "dimensions" +queryprofile[2].property[8].value "all" +queryprofile[2].property[9].name "flickr" +queryprofile[2].property[9].value "1" +queryprofile[2].property[10].name "ocr" +queryprofile[2].property[10].value "yes" +queryprofile[3].id "imageProfileTW" +queryprofile[3].inherit[1] +queryprofile[3].inherit[0] imageProfileBase +queryprofile[3].property[4] +queryprofile[3].property[0].name "hits" +queryprofile[3].property[0].value "4" +queryprofile[3].property[1].name "custid2" +queryprofile[3].property[1].value "tw" +queryprofile[3].property[2].name "language" +queryprofile[3].property[2].value "zh-Hant" +queryprofile[3].property[3].name "mlr" +queryprofile[3].property[3].value "yes"
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml1/illegalSetting.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml1/illegalSetting.xml new file mode 100644 index 00000000000..cb1592e405b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml1/illegalSetting.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="illegalSetting" type="native"> + <field name="model.notDeclared">value</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml2/unparseable.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml2/unparseable.xml new file mode 100644 index 00000000000..4bc6cb4e464 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml2/unparseable.xml @@ -0,0 +1,2 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id=""...kjh diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml3/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml3/default.xml new file mode 100644 index 00000000000..f0774e0343f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/invalidxml3/default.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="MyProfile"> + +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/production.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/production.xml new file mode 100644 index 00000000000..5d55d9626b9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/production.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="production"> + + <field name="presentation.summary">production</field> + +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us-0.2.4.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us-0.2.4.xml new file mode 100644 index 00000000000..2946f581533 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us-0.2.4.xml @@ -0,0 +1,14 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="twitter_dd-us:0.2.4" inherits = "production"> + <field name="hits">3</field> + <field name="displayGuideline">true</field> + <field name="ranking.profile">freshness-mlrrecency4</field> + <field name="qrdedup">cosine</field> + <field name="model.filter">+yst_tweet_adult_score:0</field> + <field name="blender.customer">twitter_dd</field> + <field name="reorder">-created_at</field> + <field name="filters.tweetAge">21600</field><!-- 21600 sec = 6 hours --> + <field name="resultgroupTag">true</field> + <field name="filters.userSpamScore">52</field> + <field name="filters.tweetLanguage">en</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us.xml new file mode 100644 index 00000000000..c96918f97f8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd-us.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="twitter_dd-us" inherits ="twitter_dd"> + <field name="model.filter">+yst_tweet_language:en +yst_tweet_adult_score:0</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd.xml new file mode 100644 index 00000000000..588229d21c6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/klee/twitter_dd.xml @@ -0,0 +1,14 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="twitter_dd" inherits = "production"> + <field name="hits">3</field> + <field name="displayGuideline">true</field> + <field name="ranking.profile">unranked</field> + <field name="qrdedup">user,cosine</field> + <field name="model.filter">+yst_tweet_adult_score:0</field> + <field name="blender.customer">twitter_dd</field> + <field name="reorder"></field> + <field name="ranking.sorting">-created_at</field> + <field name="filters.tweetAge">21600</field><!-- 21600 sec = 6 hours --> + <field name="resultgroupTag">true</field> + <field name="filters.userSpamScore">52</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofile1.xml new file mode 100644 index 00000000000..37354db337a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofile1.xml @@ -0,0 +1,30 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="multiprofile1" inherits="multiprofileDimensions"> <!-- A regular profile may define "virtual" children within itself --> + + <!-- Values may be set in the profile itself as usual, this becomes the default values given no matching + virtual variant provides a value for the property --> + <field name="a">general-a</field> + + <!-- The "for" attribute in a child profile supplies values in order for each of the dimensions --> + <query-profile for="us,nok ia,test1"> + <field name="a">us-nokia-test1-a</field> + </query-profile> + + <!-- Same as [us,*,*] - trailing "*"'s may be omitted --> + <query-profile for="us"> + <field name="a">us-a</field> + <field name="b">us-b</field> + </query-profile> + + <!-- Given a request which matches both the below, the one which specifies concrete values to the left + gets precedence over those specifying concrete values to the right (i.e the first one gets precedence here) --> + <query-profile for="us,nok ia,*" inherits="parent1 parent2"> + <field name="a">us-nokia-a</field> + <field name="b">us-nokia-b</field> + </query-profile> + <query-profile for="us,*,test1"> + <field name="a">us-test1-a</field> + <field name="b">us-test1-b</field> + </query-profile> + +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofileDimensions.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofileDimensions.xml new file mode 100644 index 00000000000..bfb1c08c9e8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/multiprofileDimensions.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="multiprofileDimensions"> + <!-- Names of the request parameters defining the variant profiles of this. Order matters as described below. + Each individual value looked up in this profile is resolved from the most specific matching virtual + variant profile --> + <dimensions>region,model,bucket</dimensions> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent1.xml new file mode 100644 index 00000000000..a89701a5720 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent1.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="parent1"> + <field name="parent1">parent1-value</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent2.xml new file mode 100644 index 00000000000..59f08b3ef4c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/multiprofile/parent2.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="parent2"> + <field name="parent2">parent2-value</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/default.xml new file mode 100644 index 00000000000..a1bd6a57727 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/default.xml @@ -0,0 +1,8 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> +<dimensions>dim1,dim2,dim3</dimensions> +<!-- Default values --> +<query-profile for="yahoo,uk" inherits="yahoo/uk"/> +<!--Special cases --> +<query-profile for="yahoo,uk,test" inherits="yahoo/uk/test"/> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk.xml new file mode 100644 index 00000000000..2e4dbce956d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="yahoo/uk"> +<field name="a.b">b</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk_test.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk_test.xml new file mode 100644 index 00000000000..fe4ed037532 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/news/yahoo_uk_test.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="yahoo/uk/test"> +<field name="a.c">c</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/default.xml new file mode 100644 index 00000000000..2dc8aebd6ce --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/default.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + + <dimensions>custid_1,custid_2</dimensions> + + <!-- Default values --> + <field name="ranking.profile">usrank</field> + + <query-profile for="parent" inherits="parent" /> + <query-profile for="parent,child" > + <field name="ranking.features.b">0.1</field> + </query-profile> +</query-profile> + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/parent.xml new file mode 100644 index 00000000000..157b5ae9702 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase1/parent.xml @@ -0,0 +1,5 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="parent"> + <field name="ranking.features.b">0.0</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/default.xml new file mode 100644 index 00000000000..5c2ed77211f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/default.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + + <dimensions>custid_1,custid_2,custid_3,custid_4,custid_5,custid_6</dimensions> + + <!-- Default values --> + <field name="a.profile">usrank</field> + + <query-profile for="parent" inherits="parent" /> + <query-profile for="parent,child" > + <field name="a.features.b">0.1</field> + </query-profile> +</query-profile> + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/parent.xml new file mode 100644 index 00000000000..25bff4ada59 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase2/parent.xml @@ -0,0 +1,5 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="parent"> + <field name="a.features.b">0.0</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/default.xml new file mode 100644 index 00000000000..736ab0020d6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/default.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + + <dimensions>custid_1,custid_2,custid_3,custid_4,custid_5,custid_6</dimensions> + + <!-- Default values --> + <field name="ranking.profile">usrank</field> + + <query-profile for="parent" inherits="parent" /> + <query-profile for="parent,child" > + <field name="a.features">0.1</field> + </query-profile> +</query-profile> + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/parent.xml new file mode 100644 index 00000000000..473fbd9610e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase3/parent.xml @@ -0,0 +1,5 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="parent"> + <field name="a.features">0.0</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml new file mode 100644 index 00000000000..1efcd6e4d87 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/default.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + + <dimensions>custid_1,custid_2,custid_3,custid_4,custid_5,custid_6</dimensions> + + <!-- Default values --> + <field name="a.profile">usrank</field> + + <query-profile for="parent" inherits="parent" /> + <query-profile for="parent,child" > + <field name="ranking.features">0.1</field> + </query-profile> +</query-profile> + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml new file mode 100644 index 00000000000..23c2b657182 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newscase4/parent.xml @@ -0,0 +1,5 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="parent"> + <field name="ranking.features">0.0</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/backend_news.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/backend_news.xml new file mode 100644 index 00000000000..3585ccd5eda --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/backend_news.xml @@ -0,0 +1,9 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="backend/news"> + <dimensions>vertical,sort,offset,resulttypes,rss,age,intl,testid</dimensions> + <query-profile for="news,*,*,article,0"> + <field name="discovery">sources</field> + <field name="discoverytypes">article</field> + <field name="discovery.sources.count">13</field> + </query-profile> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/default.xml new file mode 100644 index 00000000000..d8dbe6e929a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe/default.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + <field name="source.news"><ref>backend/news</ref></field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/backend.news.provider.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/backend.news.provider.xml new file mode 100644 index 00000000000..b0168f583b4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/backend.news.provider.xml @@ -0,0 +1,8 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="backend/news/provider"> + <dimensions>mode</dimensions> + <field name="provider">news_basic</field> + <query-profile for="adv"> + <field name="provider">news_adv</field> + </query-profile> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/default.xml new file mode 100644 index 00000000000..d43538bd106 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/newsfe2/default.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + <dimensions>intl</dimensions> + <query-profile for="tw" inherits="backend/news/provider"/> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants-configuration.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants-configuration.cfg new file mode 100644 index 00000000000..21d036080e1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants-configuration.cfg @@ -0,0 +1,95 @@ +queryprofile[5] +queryprofile[0].id "variants1" +queryprofile[0].dimensions[2] +queryprofile[0].dimensions[0] x +queryprofile[0].dimensions[1] y +queryprofile[0].property[2] +queryprofile[0].property[0].name "a" +queryprofile[0].property[0].value "a-deflt" +queryprofile[0].property[1].name "model.defaultIndex" +queryprofile[0].property[1].value "defaultIndex-default" +queryprofile[0].queryprofilevariant[3] +queryprofile[0].queryprofilevariant[0].fordimensionvalues[2] +queryprofile[0].queryprofilevariant[0].fordimensionvalues[0] "x1" +queryprofile[0].queryprofilevariant[0].fordimensionvalues[1] "y1" +queryprofile[0].queryprofilevariant[0].inherit[2] +queryprofile[0].queryprofilevariant[0].inherit[0] "parent1" +queryprofile[0].queryprofilevariant[0].inherit[1] "parent2" +queryprofile[0].queryprofilevariant[0].property[3] +queryprofile[0].queryprofilevariant[0].property[0].name "a" +queryprofile[0].queryprofilevariant[0].property[0].value "x1.y1.a" +queryprofile[0].queryprofilevariant[0].property[1].name "b" +queryprofile[0].queryprofilevariant[0].property[1].value "x1.y1.b" +queryprofile[0].queryprofilevariant[0].property[2].name "model.defaultIndex" +queryprofile[0].queryprofilevariant[0].property[2].value "x1.y1.defaultIndex" +queryprofile[0].queryprofilevariant[1].fordimensionvalues[1] +queryprofile[0].queryprofilevariant[1].fordimensionvalues[0] "x1" +queryprofile[0].queryprofilevariant[1].property[2] +queryprofile[0].queryprofilevariant[1].property[0].name "a" +queryprofile[0].queryprofilevariant[1].property[0].value "x1.y?.a" +queryprofile[0].queryprofilevariant[1].property[1].name "model.defaultIndex" +queryprofile[0].queryprofilevariant[1].property[1].value "x1.y?.defaultIndex" +queryprofile[0].queryprofilevariant[1].reference[1] +queryprofile[0].queryprofilevariant[1].reference[0].name "toVariants" +queryprofile[0].queryprofilevariant[1].reference[0].value "variants3" +queryprofile[0].queryprofilevariant[2].fordimensionvalues[2] +queryprofile[0].queryprofilevariant[2].fordimensionvalues[0] "*" +queryprofile[0].queryprofilevariant[2].fordimensionvalues[1] "y1" +queryprofile[0].queryprofilevariant[2].property[3] +queryprofile[0].queryprofilevariant[2].property[0].name "a" +queryprofile[0].queryprofilevariant[2].property[0].value "x?.y1.a" +queryprofile[0].queryprofilevariant[2].property[1].name "model.filter" +queryprofile[0].queryprofilevariant[2].property[1].value "x?.y1.filter" +queryprofile[0].queryprofilevariant[2].property[2].name "model.defaultIndex" +queryprofile[0].queryprofilevariant[2].property[2].value "x?.y1.defaultIndex" +queryprofile[0].queryprofilevariant[2].reference[1] +queryprofile[0].queryprofilevariant[2].reference[0].name "toVariants" +queryprofile[0].queryprofilevariant[2].reference[0].value "variants2" +queryprofile[1].id "variants2" +queryprofile[1].dimensions[2] +queryprofile[1].dimensions[0] x +queryprofile[1].dimensions[1] y +queryprofile[1].property[1] +queryprofile[1].property[0].name "c" +queryprofile[1].property[0].value "variant2:c-df" +queryprofile[1].queryprofilevariant[2] +queryprofile[1].queryprofilevariant[0].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[0].fordimensionvalues[0] "*" +queryprofile[1].queryprofilevariant[0].fordimensionvalues[1] "y1" +queryprofile[1].queryprofilevariant[0].property[1] +queryprofile[1].queryprofilevariant[0].property[0].name "c" +queryprofile[1].queryprofilevariant[0].property[0].value "variant2:y1.c" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "*" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[1] "y2" +queryprofile[1].queryprofilevariant[1].property[1] +queryprofile[1].queryprofilevariant[1].property[0].name "c" +queryprofile[1].queryprofilevariant[1].property[0].value "variant2:y2.c" +queryprofile[2].id "variants3" +queryprofile[2].dimensions[2] +queryprofile[2].dimensions[0] x +queryprofile[2].dimensions[1] y +queryprofile[2].property[1] +queryprofile[2].property[0].name "c" +queryprofile[2].property[0].value "variant3:c-df" +queryprofile[2].queryprofilevariant[2] +queryprofile[2].queryprofilevariant[0].fordimensionvalues[2] +queryprofile[2].queryprofilevariant[0].fordimensionvalues[0] "*" +queryprofile[2].queryprofilevariant[0].fordimensionvalues[1] "y1" +queryprofile[2].queryprofilevariant[0].property[1] +queryprofile[2].queryprofilevariant[0].property[0].name "c" +queryprofile[2].queryprofilevariant[0].property[0].value "variant3:y1.c" +queryprofile[2].queryprofilevariant[1].fordimensionvalues[2] +queryprofile[2].queryprofilevariant[1].fordimensionvalues[0] "*" +queryprofile[2].queryprofilevariant[1].fordimensionvalues[1] "y2" +queryprofile[2].queryprofilevariant[1].property[1] +queryprofile[2].queryprofilevariant[1].property[0].name "c" +queryprofile[2].queryprofilevariant[1].property[0].value "variant3:y2.c" +queryprofile[3].id "parent1" +queryprofile[3].property[1] +queryprofile[3].property[0].name "parent1" +queryprofile[3].property[0].value "parent1-value" +queryprofile[4].id "parent2" +queryprofile[4].property[1] +queryprofile[4].property[0].name "parent2" +queryprofile[4].property[0].value "parent2-value" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg new file mode 100644 index 00000000000..ec091ecf2ea --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profile-variants2.cfg @@ -0,0 +1,63 @@ +queryprofile[4] +queryprofile[0].id "default" +queryprofile[0].property[5] +queryprofile[0].property[0].name "hits" +queryprofile[0].property[0].value "5" +queryprofile[0].property[1].name "model.defaultIndex" +queryprofile[0].property[1].value "title" +queryprofile[0].property[2].name "ranking.features.query(scorelimit)" +queryprofile[0].property[2].value "-20" +queryprofile[0].property[3].name "ranking.profile" +queryprofile[0].property[3].value "production1" +queryprofile[0].property[4].name "ranking.properties.dotProduct.X" +queryprofile[0].property[4].value "(a:1,b:2)" +queryprofile[1].id "multi" +queryprofile[1].inherit[1] +queryprofile[1].inherit[0] "default" +queryprofile[1].dimensions[2] +queryprofile[1].dimensions[0] "myquery" +queryprofile[1].dimensions[1] "myindex" +queryprofile[1].reference[1] +queryprofile[1].reference[0].name "model" +queryprofile[1].reference[0].value "querybest" +queryprofile[1].property[1] +queryprofile[1].property[0].name "model.defaultIndex" +queryprofile[1].property[0].value "default-default" +queryprofile[1].queryprofilevariant[3] +queryprofile[1].queryprofilevariant[0].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[0].fordimensionvalues[0] "lo ve" +queryprofile[1].queryprofilevariant[0].fordimensionvalues[1] "default" +queryprofile[1].queryprofilevariant[0].reference[1] +queryprofile[1].queryprofilevariant[0].reference[0].name "model" +queryprofile[1].queryprofilevariant[0].reference[0].value "querylove" +queryprofile[1].queryprofilevariant[0].property[1] +queryprofile[1].queryprofilevariant[0].property[0].name "model.defaultIndex" +queryprofile[1].queryprofilevariant[0].property[0].value "default" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[1].fordimensionvalues[0] "*" +queryprofile[1].queryprofilevariant[1].fordimensionvalues[1] "default" +queryprofile[1].queryprofilevariant[1].property[1] +queryprofile[1].queryprofilevariant[1].property[0].name "model.defaultIndex" +queryprofile[1].queryprofilevariant[1].property[0].value "default" +queryprofile[1].queryprofilevariant[2].fordimensionvalues[2] +queryprofile[1].queryprofilevariant[2].fordimensionvalues[0] "lo ve" +queryprofile[1].queryprofilevariant[2].fordimensionvalues[1] "*" +queryprofile[1].queryprofilevariant[2].reference[1] +queryprofile[1].queryprofilevariant[2].reference[0].name "model" +queryprofile[1].queryprofilevariant[2].reference[0].value "querylove" +queryprofile[2].id "querybest" +queryprofile[2].type "model" +queryprofile[2].property[2] +queryprofile[2].property[0].name "defaultIndex" +queryprofile[2].property[0].value "title" +queryprofile[2].property[1].name "queryString" +queryprofile[2].property[1].value "best" +queryprofile[2].property[1].overridable false +queryprofile[3].id "querylove" +queryprofile[3].type "model" +queryprofile[3].property[2] +queryprofile[3].property[0].name "defaultIndex" +queryprofile[3].property[0].value "title" +queryprofile[3].property[1].name "queryString" +queryprofile[3].property[1].value "love" +queryprofile[3].property[1].overridable false diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profiles-configuration.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profiles-configuration.cfg new file mode 100644 index 00000000000..6d3e957a722 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/query-profiles-configuration.cfg @@ -0,0 +1,52 @@ +queryprofile[6] +queryprofile[0].id "subprofile3" +queryprofile[0].property[1] +queryprofile[0].property[0].name "e" +queryprofile[0].property[0].value "e-value-subprofile3" +queryprofile[1].id "inherited1" +queryprofile[1].property[4] +queryprofile[1].property[0].name "a" +queryprofile[1].property[0].value "a-value-inherited1" +queryprofile[1].property[1].name "e" +queryprofile[1].property[1].value "e-value-inherited1" +queryprofile[1].property[2].name "c.d" +queryprofile[1].property[2].value "c.d-value-inherited1" +queryprofile[1].property[3].name "g.d2" +queryprofile[1].property[3].value "g.d2-value-inherited1" +queryprofile[2].id "inherited2" +queryprofile[2].property[2] +queryprofile[2].property[0].name "a" +queryprofile[2].property[0].value "a-value-inherited2" +queryprofile[2].property[1].name "c.d2" +queryprofile[2].property[1].value "c.d2-value-inherited2" +queryprofile[3].id "subprofile1" +queryprofile[3].property[2] +queryprofile[3].property[0].name "a" +queryprofile[3].property[0].value "a-value-subprofile1" +queryprofile[3].property[1].name "c.d" +queryprofile[3].property[1].value "c.d-value-subprofile1" +queryprofile[4].id "subprofile2" +queryprofile[4].property[2] +queryprofile[4].property[0].name "a" +queryprofile[4].property[0].value "a-value-subprofile2" +queryprofile[4].property[1].name "c.d" +queryprofile[4].property[1].value "c.d-value-subprofile2" +queryprofile[5].id "default" +queryprofile[5].inherit[2] +queryprofile[5].inherit[0] inherited1 +queryprofile[5].inherit[1] inherited2 +queryprofile[5].property[3] +queryprofile[5].property[0].name "a" +queryprofile[5].property[0].value "a-value" +queryprofile[5].property[1].name "b" +queryprofile[5].property[1].value "b-value" +queryprofile[5].property[2].name "c.d" +queryprofile[5].property[2].value "c.d-value" +queryprofile[5].property[2].overridable "false" +queryprofile[5].reference[3] +queryprofile[5].reference[0].name "sub1" +queryprofile[5].reference[0].value "subprofile1" +queryprofile[5].reference[1].name "sub2" +queryprofile[5].reference[1].value "subprofile2" +queryprofile[5].reference[2].name "g" +queryprofile[5].reference[2].value "subprofile3" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/default.xml new file mode 100644 index 00000000000..ce40b67e3de --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/default.xml @@ -0,0 +1,10 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + <field name="hits">5</field> + <field name="model.defaultIndex">%{subst}%{subst.end}</field> + <field name="ranking.profile">production1</field> + <field name="ranking.features.query(scorelimit)">-20</field> + <field name="ranking.properties.dotProduct.X">(a:1,b:2)</field> + <field name="subst">tit</field> + <field name="subst.end">le</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatory.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatory.xml new file mode 100644 index 00000000000..5b67e6682c8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatory.xml @@ -0,0 +1,3 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id ="mandatory" type="mandatory"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatorySpecified.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatorySpecified.xml new file mode 100644 index 00000000000..39b835cb536 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/mandatorySpecified.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id ="mandatorySpecified" type="mandatory"> + <field name="timeout">1377</field> + <field name="foo">37</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multi.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multi.xml new file mode 100644 index 00000000000..a4feef724b4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multi.xml @@ -0,0 +1,22 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="multi" inherits="default multiDimensions"> <!-- default sets default-index to title --> + <field name="model"><ref>querybest</ref></field> + + <query-profile for="love,default"> + <field name="model"><ref>querylove</ref></field> + <field name="model.defaultIndex">default</field> + </query-profile> + + <query-profile for="*,default"> + <field name="model.defaultIndex">default</field> + </query-profile> + + <query-profile for="love"> + <field name="model"><ref>querylove</ref></field> + </query-profile> + + <query-profile for="inheritslove" inherits="rootWithFilter"> + <field name="model.filter">+me</field> + </query-profile> + +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multiDimensions.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multiDimensions.xml new file mode 100644 index 00000000000..e4abc4a2202 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/multiDimensions.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="multiDimensions"> + <dimensions>myquery, myindex </dimensions> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querybest.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querybest.xml new file mode 100644 index 00000000000..b6e5031705b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querybest.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="querybest" type="model"> + <field name="defaultIndex">title</field> + <field name="queryString" overridable="false">best</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querylove.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querylove.xml new file mode 100644 index 00000000000..e7864977804 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/querylove.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="querylove" type="model"> + <field name="defaultIndex">title</field> + <field name="queryString" overridable="false">love</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/referingQuerybest.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/referingQuerybest.xml new file mode 100644 index 00000000000..ceb1d0302c6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/referingQuerybest.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="referingQuerybest" inherits="default"> + <field name="model"><ref>querybest</ref></field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.unoverridableIndex.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.unoverridableIndex.xml new file mode 100644 index 00000000000..e8412121d67 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.unoverridableIndex.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="root/unoverridableIndex" type="root"> + <field name="model.defaultIndex" overridable="false">default</field> + <field name="hits">1</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.xml new file mode 100644 index 00000000000..60e5026bc5f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/root.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="root" inherits="default" type="root"> + <field name="model.defaultIndex">%{indexname}</field> + <field name="ranking.profile">test1</field> + <field name="hits">10</field> + <field name="indexname">default</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootChild.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootChild.xml new file mode 100644 index 00000000000..b7060093d74 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootChild.xml @@ -0,0 +1,3 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="rootChild" type="root" inherits="root"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootStrict.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootStrict.xml new file mode 100644 index 00000000000..f942e5c3cb5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootStrict.xml @@ -0,0 +1,3 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="rootStrict" type="rootStrict" inherits="root"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootWithFilter.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootWithFilter.xml new file mode 100644 index 00000000000..1cb98d12ba8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/rootWithFilter.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="rootWithFilter" inherits="root"> + <field name="model.filter">+best</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/test.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/test.xml new file mode 100644 index 00000000000..6146a6ef7d0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/test.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="test"> + <field name="traceLevel">3</field> + <field name="nocache">true</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/forbidding.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/forbidding.xml new file mode 100644 index 00000000000..6b1f666929f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/forbidding.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile-type id="forbidding"> + <strict/> + <field name="query" type="string"/> + <field name="model" type="query-profile:model"/> + <field name="hits" type="integer"/> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/mandatory.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/mandatory.xml new file mode 100644 index 00000000000..ea6180f7379 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/mandatory.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile-type id="mandatory" inherits="native"> + <field name="timeout" type="string" mandatory="true"/> + <field name="foo" type="integer" mandatory="true"/> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/root.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/root.xml new file mode 100644 index 00000000000..74077baafff --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/root.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile-type id="root" inherits="native"> + <match path="true"/> + <field name="indexname" type="string" alias="index-name idx"/> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/rootStrict.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/rootStrict.xml new file mode 100644 index 00000000000..9b257e1c8a6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/queryprofilevariants2/types/rootStrict.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile-type id="rootStrict" inherits="root"> + <strict/> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile1.xml new file mode 100644 index 00000000000..4b81164309d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile1.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="MyProfile1" > + <field name="name">MyProfile1</field> + <field name="myProfile1Only">myProfile1Only</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile2.xml new file mode 100644 index 00000000000..14bb544b744 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/MyProfile2.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="MyProfile2" > + <field name="name">MyProfile2</field> + <field name="myProfile2Only">myProfile2Only</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/default.xml new file mode 100644 index 00000000000..252b84600ed --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverride/default.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default" > + <field name="name">default</field> + <field name="profileRef"><ref>MyProfile1</ref></field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile1.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile1.xml new file mode 100644 index 00000000000..4b81164309d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile1.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="MyProfile1" > + <field name="name">MyProfile1</field> + <field name="myProfile1Only">myProfile1Only</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile2.xml new file mode 100644 index 00000000000..14bb544b744 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/MyProfile2.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="MyProfile2" > + <field name="name">MyProfile2</field> + <field name="myProfile2Only">myProfile2Only</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/default.xml new file mode 100644 index 00000000000..cf425a819e0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/default.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default" type="default"> + <field name="name">default</field> + <field name="profileRef"><ref>MyProfile1</ref></field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/types/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/types/default.xml new file mode 100644 index 00000000000..78a1db68661 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/refoverridetyped/types/default.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile-type id="default"> + <field name="profileRef" type="query-profile"/> +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/common.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/common.xml new file mode 100755 index 00000000000..eb6cc805bc4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/common.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="common"> + <dimensions>source.common.intl</dimensions> + <query-profile for="tw" inherits="provider"/> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/myprofile.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/myprofile.xml new file mode 100755 index 00000000000..6260c08e9fa --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/myprofile.xml @@ -0,0 +1,6 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="myprofile"> + <field name="sources">common</field> + <field name="source"><ref>source</ref></field> +</query-profile> + diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/provider.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/provider.xml new file mode 100755 index 00000000000..71ed8000a9d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/provider.xml @@ -0,0 +1,9 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="provider" > + <dimensions>source.common.mode</dimensions> + <field name="provider">yst</field> + <query-profile for="adv"> + <field name="provider">news</field> + </query-profile> + <field name="dummy">test</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/source.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/source.xml new file mode 100755 index 00000000000..a33a3cb455a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/sourceprovider/source.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="source"> + <field name="common"><ref>common</ref></field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/systemtest/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/systemtest/default.xml new file mode 100644 index 00000000000..873233042e4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/systemtest/default.xml @@ -0,0 +1,8 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default"> + <field name="foo">bar</field> + <field name="hits">5</field> + <field name="model.defaultIndex">%{subst}%{subst.end}</field> + <field name="subst">tit</field> + <field name="subst.end">le</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg new file mode 100644 index 00000000000..27964bbd94f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed-profiles.cfg @@ -0,0 +1,42 @@ +queryprofiletype[4] + +queryprofiletype[0].id testtype +queryprofiletype[0].field[7] +queryprofiletype[0].field[0].name "myString" +queryprofiletype[0].field[0].type "string" +queryprofiletype[0].field[0].mandatory true +queryprofiletype[0].field[1].name "myInteger" +queryprofiletype[0].field[1].type "integer" +queryprofiletype[0].field[1].overridable false +queryprofiletype[0].field[2].name "myLong" +queryprofiletype[0].field[2].type "long" +queryprofiletype[0].field[3].name "myFloat" +queryprofiletype[0].field[3].type "float" +queryprofiletype[0].field[4].name "myDouble" +queryprofiletype[0].field[4].type "double" +queryprofiletype[0].field[5].name "myQueryProfile" +queryprofiletype[0].field[5].type "query-profile" +queryprofiletype[0].field[6].name "myUserQueryProfile" +queryprofiletype[0].field[6].type "query-profile:user" +queryprofiletype[0].field[6].alias "myqp user-profile" + +queryprofiletype[1].id testtypestrict +queryprofiletype[1].strict true +queryprofiletype[1].matchaspath true +queryprofiletype[1].inherit[1] +queryprofiletype[1].inherit[0] "testtype" +queryprofiletype[1].field[1] +queryprofiletype[1].field[0].name "myUserQueryProfile" +queryprofiletype[1].field[0].type "query-profile:userstrict" + +queryprofiletype[2].id user +queryprofiletype[2].field[2] +queryprofiletype[2].field[0].name "myUserString" +queryprofiletype[2].field[0].type "string" +queryprofiletype[2].field[1].name "myUserInteger" +queryprofiletype[2].field[1].type "integer" + +queryprofiletype[3].id userstrict +queryprofiletype[3].strict true +queryprofiletype[3].inherit[1] +queryprofiletype[3].inherit[0] "user" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/.gitignore b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/.gitignore new file mode 100644 index 00000000000..0a1c2c442c1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/.gitignore @@ -0,0 +1,10 @@ +bundles.cfg +container-mbus.cfg +diagnostics.cfg +documentdb-info.cfg +documentmanager.cfg +int.cfg +qr-templates.cfg +renderers.cfg +schemamapping.cfg +string.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/chains.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/chains.cfg new file mode 100644 index 00000000000..99af9283ea8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/chains.cfg @@ -0,0 +1,16 @@ +chains[2] +chains[0].id test +chains[0].components[3] +chains[0].components[0] SettingSearcher +chains[0].components[1] ReceivingSearcher +chains[0].components[2] TestSearcher +chains[1].id default +chains[1].components[3] +chains[1].components[0] SettingSearcher +chains[1].components[1] ReceivingSearcher +chains[1].components[2] DefaultSearcher +components[4] +components[0].id SettingSearcher +components[1].id ReceivingSearcher +components[2].id TestSearcher +components[3].id DefaultSearcher diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg new file mode 100644 index 00000000000..1110d76f887 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/components.cfg @@ -0,0 +1,11 @@ +components[6] +components[0].id SettingSearcher +components[0].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$SettingSearcher +components[1].id ReceivingSearcher +components[1].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$ReceivingSearcher +components[2].id TestSearcher +components[2].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$TestSearcher +components[3].id DefaultSearcher +components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$DefaultSearcher +components[4].id com.yahoo.search.handler.SearchHandler +components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/handlers.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/index-info.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/index-info.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-search.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/query-profiles.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/query-profiles.cfg new file mode 100644 index 00000000000..7c0b22a3606 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/query-profiles.cfg @@ -0,0 +1,49 @@ +queryprofile[5] +queryprofile[0].id "default" +queryprofile[1].type "root" +queryprofile[0].property[1] +queryprofile[0].property[0].name "searchChain" +queryprofile[0].property[0].value "test" +queryprofile[1].id "test" +queryprofile[1].type "root" +queryprofile[1].property[1] +queryprofile[1].property[0].name "searchChain" +queryprofile[1].property[0].value "default" +queryprofile[2].id "hitsoffset" +queryprofile[2].type "root" +queryprofile[2].property[2] +queryprofile[2].property[0].name "hits" +queryprofile[2].property[0].value "22" +queryprofile[2].property[1].name "offset" +queryprofile[2].property[1].value "80" +queryprofile[3].id "root" +queryprofile[3].type "root" +queryprofile[3].property[2] +queryprofile[3].property[0].name "sub.value1" +queryprofile[3].property[0].value "subvalue1" +queryprofile[3].property[1].name "sub.value2" +queryprofile[3].property[1].value "subvalue2" +queryprofile[4].id "newsub" +queryprofile[4].type "sub" +queryprofile[4].property[2] +queryprofile[4].property[0].name "value1" +queryprofile[4].property[0].value "newsubvalue1" +queryprofile[4].property[1].name "value2" +queryprofile[4].property[1].value "newsubvalue2" + +queryprofiletype[2] +queryprofiletype[0].id "root" +queryprofiletype[0].inherit[1] +queryprofiletype[0].inherit[0] "native" +queryprofiletype[0].strict true +queryprofiletype[0].matchaspath true +queryprofiletype[0].field[1] +queryprofiletype[0].field[0].name "sub" +queryprofiletype[0].field[0].type "query-profile:sub" +queryprofiletype[1].id "sub" +queryprofiletype[1].strict true +queryprofiletype[1].field[2] +queryprofiletype[1].field[0].name "value1" +queryprofiletype[1].field[0].type "string" +queryprofiletype[1].field[1].name "value2" +queryprofiletype[1].field[1].type "string" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/typed/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/.gitignore b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/.gitignore new file mode 100644 index 00000000000..0a1c2c442c1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/.gitignore @@ -0,0 +1,10 @@ +bundles.cfg +container-mbus.cfg +diagnostics.cfg +documentdb-info.cfg +documentmanager.cfg +int.cfg +qr-templates.cfg +renderers.cfg +schemamapping.cfg +string.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/chains.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/chains.cfg new file mode 100644 index 00000000000..99af9283ea8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/chains.cfg @@ -0,0 +1,16 @@ +chains[2] +chains[0].id test +chains[0].components[3] +chains[0].components[0] SettingSearcher +chains[0].components[1] ReceivingSearcher +chains[0].components[2] TestSearcher +chains[1].id default +chains[1].components[3] +chains[1].components[0] SettingSearcher +chains[1].components[1] ReceivingSearcher +chains[1].components[2] DefaultSearcher +components[4] +components[0].id SettingSearcher +components[1].id ReceivingSearcher +components[2].id TestSearcher +components[3].id DefaultSearcher diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg new file mode 100644 index 00000000000..70f5452b74d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/components.cfg @@ -0,0 +1,11 @@ +components[6] +components[0].id SettingSearcher +components[0].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$SettingSearcher +components[1].id ReceivingSearcher +components[1].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$ReceivingSearcher +components[2].id TestSearcher +components[2].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$TestSearcher +components[3].id DefaultSearcher +components[3].classId com.yahoo.search.query.profile.config.test.QueryProfileIntegrationTestCase$DefaultSearcher +components[4].id com.yahoo.search.handler.SearchHandler +components[5].id com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/handlers.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/index-info.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/index-info.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-search.cfg diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/query-profiles.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/query-profiles.cfg new file mode 100644 index 00000000000..e5fe53e4e0a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/query-profiles.cfg @@ -0,0 +1,29 @@ +queryprofile[5] +queryprofile[0].id "default" +queryprofile[0].property[1] +queryprofile[0].property[0].name "searchChain" +queryprofile[0].property[0].value "test" +queryprofile[1].id "test" +queryprofile[1].property[2] +queryprofile[1].property[0].name "searchChain" +queryprofile[1].property[0].value "default" +queryprofile[1].property[1].name "model.defaultIndex" +queryprofile[1].property[1].value "index" +queryprofile[2].id "hitsoffset" +queryprofile[2].property[2] +queryprofile[2].property[0].name "hits" +queryprofile[2].property[0].value "20" +queryprofile[2].property[1].name "offset" +queryprofile[2].property[1].value "80" +queryprofile[3].id "root" +queryprofile[3].property[2] +queryprofile[3].property[0].name "sub.value1" +queryprofile[3].property[0].value "subvalue1" +queryprofile[3].property[1].name "sub.value2" +queryprofile[3].property[1].value "subvalue2" +queryprofile[4].id "newsub" +queryprofile[4].property[2] +queryprofile[4].property[0].name "value1" +queryprofile[4].property[0].value "newsubvalue1" +queryprofile[4].property[1].name "value2" +queryprofile[4].property[1].value "newsubvalue2" diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/untyped/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/default.xml new file mode 100644 index 00000000000..a93771e68cb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/default.xml @@ -0,0 +1,10 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<query-profile id="default"> + + <field name="hits" overridable="false">20</field> + <field name="user.trusted" overridable="false">false</field> + <field name="model.defaultIndex" overridable="false">default</field> + +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/modelSettings.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/modelSettings.xml new file mode 100644 index 00000000000..a045635966e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/modelSettings.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="modelSettings" type="model"> + <field name="queryString">some query</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/referencingModelSettings.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/referencingModelSettings.xml new file mode 100644 index 00000000000..fe07ae55a1f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/referencingModelSettings.xml @@ -0,0 +1,7 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="referencingModelSettings"> + + <field name="model"><ref>modelSettings</ref></field> + <field name="model.defaultIndex">aDefaultIndex</field> + +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml new file mode 100644 index 00000000000..05606e2be8d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/root.xml @@ -0,0 +1,9 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<query-profile id="root" type="rootType"> + + <field name="hits">30</field> + <field name="traceLevel">3</field> + +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/someUser.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/someUser.xml new file mode 100644 index 00000000000..42175fd6368 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/someUser.xml @@ -0,0 +1,12 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<query-profile id="someUser" type="user"> + <dimensions>x</dimensions> + <field name="age">18</field> + <field name="sub.test">5</field> + <query-profile for="x1"> + <field name="age">37</field> + <field name="gender">male</field> + </query-profile> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/rootType.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/rootType.xml new file mode 100644 index 00000000000..bf49c03e57f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/rootType.xml @@ -0,0 +1,12 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<query-profile-type id="rootType" inherits="native"> + + <strict/> + <match path="true"/> + + <field name="time" type="long" mandatory="true"/> + <field name="user" type="query-profile:user"/> + +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/user.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/user.xml new file mode 100644 index 00000000000..96f702a43eb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/validxml/types/user.xml @@ -0,0 +1,10 @@ +<?xml version="1.0"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +<query-profile-type id="user"> + + <field name="age" type="integer" mandatory="true" alias="alder anno"/> + <field name="gender" type="string" alias="sex"/> + <field name="robot" type="boolean" overridable="false"/> + +</query-profile-type> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.0.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.0.xml new file mode 100644 index 00000000000..7aad3ee2df7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.0.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="MyProfile:1.0.0"> + <field name="name">MyProfile:1.0.0</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.a.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.a.xml new file mode 100644 index 00000000000..0314223b732 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.a.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="MyProfile:1.0.2.a"> + <field name="name">MyProfile:1.0.2.a</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.xml new file mode 100644 index 00000000000..debd93850ae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/MyProfile-1.0.2.xml @@ -0,0 +1,4 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="MyProfile:1.0.2"> + <field name="name">MyProfile:1.0.2</field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/default.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/default.xml new file mode 100644 index 00000000000..67dc76c851f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versionrefs/default.xml @@ -0,0 +1,5 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="default" > + <field name="name">default</field> + <field name="profile1"><ref>MyProfile</ref></field> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile-1.20.100.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile-1.20.100.xml new file mode 100644 index 00000000000..08ae7dd9ee0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile-1.20.100.xml @@ -0,0 +1,3 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="testprofile:1.20.100"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile.xml b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile.xml new file mode 100644 index 00000000000..12309a78075 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/versions/testprofile.xml @@ -0,0 +1,3 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<query-profile id="testprofile"> +</query-profile> diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java new file mode 100644 index 00000000000..4e68ac92adc --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/CloningTestCase.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.jdisc.http.HttpRequest.Method; + +/** + * @author bratseth + */ +public class CloningTestCase extends junit.framework.TestCase { + + public void testCloningWithVariants() { + QueryProfile test = new QueryProfile("test"); + test.setDimensions(new String[] {"x"} ); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q&x=x1", Method.GET), test.compile(null)); + q1.properties().set("a","a1"); + Query q2 = q1.clone(); + q2.properties().set("a","a2"); + assertEquals("a1",q1.properties().get("a")); + assertEquals("a2",q2.properties().get("a")); + } + + public void testShallowCloning() { + QueryProfile test = new QueryProfile("test"); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + q1.properties().set("a",new MutableString("a1")); + Query q2 = q1.clone(); + ((MutableString)q2.properties().get("a")).set("a2"); + assertEquals("a2",q1.properties().get("a").toString()); + assertEquals("a2",q2.properties().get("a").toString()); + } + + public void testShallowCloningWithVariants() { + QueryProfile test = new QueryProfile("test"); + test.setDimensions(new String[] {"x"} ); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q&x=x1", Method.GET), test.compile(null)); + q1.properties().set("a",new MutableString("a1")); + Query q2 = q1.clone(); + ((MutableString)q2.properties().get("a")).set("a2"); + assertEquals("a2",q1.properties().get("a").toString()); + assertEquals("a2",q2.properties().get("a").toString()); + } + + public void testDeepCloning() { + QueryProfile test=new QueryProfile("test"); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + q1.properties().set("a",new CloneableMutableString("a1")); + Query q2=q1.clone(); + ((MutableString)q2.properties().get("a")).set("a2"); + assertEquals("a1",q1.properties().get("a").toString()); + assertEquals("a2",q2.properties().get("a").toString()); + } + + public void testDeepCloningWithVariants() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"} ); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q&x=x1", Method.GET), test.compile(null)); + q1.properties().set("a",new CloneableMutableString("a1")); + Query q2=q1.clone(); + ((MutableString)q2.properties().get("a")).set("a2"); + assertEquals("a1",q1.properties().get("a").toString()); + assertEquals("a2",q2.properties().get("a").toString()); + } + + public void testReAssignment() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"} ); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q&x=x1", Method.GET), test.compile(null)); + q1.properties().set("a","a1"); + q1.properties().set("a","a2"); + assertEquals("a2",q1.properties().get("a")); + } + + public void testThreeLevelsOfCloning() { + QueryProfile test = new QueryProfile("test"); + test.set("a", "config-a", (QueryProfileRegistry)null); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + + q1.properties().set("a","q1-a"); + Query q2=q1.clone(); + q2.properties().set("a","q2-a"); + Query q31=q2.clone(); + q31.properties().set("a","q31-a"); + Query q32=q2.clone(); + q32.properties().set("a","q32-a"); + + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + q2.properties().set("a","q2-a-2"); + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a-2",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + } + + public void testThreeLevelsOfCloningReverseSetOrder() { + QueryProfile test = new QueryProfile("test"); + test.set("a", "config-a", (QueryProfileRegistry)null); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + + Query q2=q1.clone(); + Query q31=q2.clone(); + Query q32=q2.clone(); + q32.properties().set("a","q32-a"); + q31.properties().set("a","q31-a"); + q2.properties().set("a","q2-a"); + q1.properties().set("a","q1-a"); + + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + q2.properties().set("a","q2-a-2"); + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a-2",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + } + + public void testThreeLevelsOfCloningMiddleFirstSetOrder1() { + QueryProfile test = new QueryProfile("test"); + test.set("a", "config-a", (QueryProfileRegistry)null); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + + Query q2=q1.clone(); + Query q31=q2.clone(); + Query q32=q2.clone(); + q2.properties().set("a","q2-a"); + q32.properties().set("a","q32-a"); + q31.properties().set("a","q31-a"); + q1.properties().set("a","q1-a"); + + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + q2.properties().set("a","q2-a-2"); + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a-2",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("q32-a",q32.properties().get("a").toString()); + } + + public void testThreeLevelsOfCloningMiddleFirstSetOrder2() { + QueryProfile test = new QueryProfile("test"); + test.set("a", "config-a", (QueryProfileRegistry)null); + test.freeze(); + Query q1 = new Query(HttpRequest.createTestRequest("?query=q", Method.GET), test.compile(null)); + + Query q2=q1.clone(); + Query q31=q2.clone(); + Query q32=q2.clone(); + q2.properties().set("a","q2-a"); + q31.properties().set("a","q31-a"); + q1.properties().set("a","q1-a"); + + assertEquals("q1-a",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("config-a",q32.properties().get("a").toString()); + q1.properties().set("a","q1-a-2"); + assertEquals("q1-a-2",q1.properties().get("a").toString()); + assertEquals("q2-a",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("config-a",q32.properties().get("a").toString()); + q2.properties().set("a","q2-a-2"); + assertEquals("q1-a-2",q1.properties().get("a").toString()); + assertEquals("q2-a-2",q2.properties().get("a").toString()); + assertEquals("q31-a",q31.properties().get("a").toString()); + assertEquals("config-a",q32.properties().get("a").toString()); + } + + public static class MutableString { + + private String string; + + public MutableString(String string) { + this.string=string; + } + + public void set(String string) { this.string=string; } + + public @Override String toString() { return string; } + + public @Override int hashCode() { return string.hashCode(); } + + public @Override boolean equals(Object other) { + if (other==this) return true; + if ( ! (other instanceof MutableString)) return false; + return ((MutableString)other).string.equals(string); + } + + } + + public static class CloneableMutableString extends MutableString implements Cloneable { + + public CloneableMutableString(String string) { + super(string); + } + + public @Override CloneableMutableString clone() { + try { + return (CloneableMutableString)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/DimensionBindingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/DimensionBindingTestCase.java new file mode 100644 index 00000000000..28cd0f4daf9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/DimensionBindingTestCase.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.search.query.profile.DimensionBinding; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class DimensionBindingTestCase { + + @Test + public void testCombining() { + assertEquals(binding("a, b, c", "a=1", "b=1", "c=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("c", "c=1"))); + + assertEquals(binding("a, b, c", "a=1", "b=1", "c=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("a, c", "a=1", "c=1"))); + + assertEquals(binding("c, a, b", "c=1", "a=1", "b=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("c, a", "a=1", "c=1"))); + + assertEquals(binding("a, b", "a=1", "b=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("a, b", "a=1", "b=1"))); + + assertEquals(DimensionBinding.invalidBinding, + binding("a, b", "a=1", "b=1").combineWith(binding("b, a", "a=1", "b=1"))); + + assertEquals(binding("a, b", "a=1", "b=1"), + binding("a, b", "a=1", "b=1").combineWith(binding("b", "b=1"))); + + assertEquals(binding("a, b, c", "a=1", "b=1", "c=1"), + binding("a, b, c", "a=1", "c=1").combineWith(binding("a, b, c", "a=1", "b=1", "c=1"))); + + assertEquals(binding("a, b, c", "a=1", "b=1", "c=1"), + binding("a, c", "a=1", "c=1").combineWith(binding("a, b, c", "a=1", "b=1", "c=1"))); + } + + // found DimensionBinding [custid_1=yahoo, custid_2=ca, custid_3=sc, custid_4=null, custid_5=null, custid_6=null], combined with DimensionBinding [custid_1=yahoo, custid_2=null, custid_3=sc, custid_4=null, custid_5=null, custid_6=null] to Invalid DimensionBinding + @Test + public void testCombiningBindingsWithNull() { + List<String> dimensions = list("a,b"); + + Map<String, String> map1 = new HashMap<>(); + map1.put("a","a1"); + map1.put("b","b1"); + + Map<String, String> map2 = new HashMap<>(); + map2.put("a","a1"); + map2.put("b",null); + + assertEquals(DimensionBinding.createFrom(dimensions, map1), + DimensionBinding.createFrom(dimensions, map1).combineWith(DimensionBinding.createFrom(dimensions, map2))); + } + + private DimensionBinding binding(String dimensions, String ... dimensionPoints) { + return DimensionBinding.createFrom(list(dimensions), QueryProfileVariantsTestCase.toMap(dimensionPoints)); + } + + private List<String> list(String listString) { + List<String> l = new ArrayList<>(); + for (String s : listString.split(",")) + l.add(s.trim()); + return l; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java new file mode 100644 index 00000000000..576605b8fa4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.search.query.profile.DumpTool; + +/** + * @author bratseth + */ +public class DumpToolTestCase extends junit.framework.TestCase { + + String profileDir="src/test/java/com/yahoo/search/query/profile/config/test/multiprofile"; + + public void testNoParameters() { + assertTrue(new DumpTool().resolveAndDump().startsWith("Dumps all resolved")); + } + + public void testHelpParameter() { + assertTrue(new DumpTool().resolveAndDump("-help").startsWith("Dumps all resolved")); + } + + public void testNoDimensionValues() { + assertTrue(new DumpTool().resolveAndDump("multiprofile1",profileDir).startsWith("a=general-a\n")); + } + + public void testAllParametersSet() { + assertTrue(new DumpTool().resolveAndDump("multiprofile1",profileDir,"").startsWith("a=general-a\n")); + } + + //This test is order dependent. Fix this!! + public void testVariant() { + System.out.println(new DumpTool().resolveAndDump("multiprofile1",profileDir,"region=us")); + assertTrue(new DumpTool().resolveAndDump("multiprofile1",profileDir,"region=us").startsWith("a=us-a\nb=us-b\nregion=us")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryFromProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryFromProfileTestCase.java new file mode 100644 index 00000000000..e67d7723932 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryFromProfileTestCase.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.QueryProfileType; + +/** + * Test using the profile to set the query to execute + * + * @author bratseth + */ +public class QueryFromProfileTestCase extends junit.framework.TestCase { + + public void testQueryFromProfile1() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile topLevel = new QueryProfile("topLevel"); + topLevel.setType(registry.getTypeRegistry().getComponent("native")); + registry.register(topLevel); + + QueryProfile queryBest = new QueryProfile("querybest"); + queryBest.setType(registry.getTypeRegistry().getComponent("model")); + queryBest.set("queryString", "best", registry); + registry.register(queryBest); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?model=querybest", Method.GET), cRegistry.getComponent("topLevel")); + assertEquals("best", query.properties().get("model.queryString")); + assertEquals("best", query.getModel().getQueryTree().toString()); + } + + public void testQueryFromProfile2() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileType rootType = new QueryProfileType("root"); + rootType.inherited().add(registry.getTypeRegistry().getComponent("native")); + registry.getTypeRegistry().register(rootType); + + QueryProfile root = new QueryProfile("root"); + root.setType(rootType); + registry.register(root); + + QueryProfile queryBest=new QueryProfile("querybest"); + queryBest.setType(registry.getTypeRegistry().getComponent("model")); + queryBest.set("queryString", "best", registry); + registry.register(queryBest); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?query=overrides&model=querybest", Method.GET), cRegistry.getComponent("root")); + assertEquals("overrides", query.properties().get("model.queryString")); + assertEquals("overrides", query.getModel().getQueryTree().toString()); + } + + public void testQueryFromProfile3() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileType rootType = new QueryProfileType("root"); + rootType.inherited().add(registry.getTypeRegistry().getComponent("native")); + registry.getTypeRegistry().register(rootType); + + QueryProfile root = new QueryProfile("root"); + root.setType(rootType); + registry.register(root); + + QueryProfile queryBest=new QueryProfile("querybest"); + queryBest.setType(registry.getTypeRegistry().getComponent("model")); + queryBest.set("queryString", "best", registry); + registry.register(queryBest); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?query=overrides&model=querybest", Method.GET), cRegistry.getComponent("root")); + assertEquals("overrides", query.properties().get("model.queryString")); + assertEquals("overrides", query.getModel().getQueryTree().toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileCloneMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileCloneMicroBenchmark.java new file mode 100644 index 00000000000..684e6072c5d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileCloneMicroBenchmark.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfileRegistry; + +/** + * @author bratseth + */ +public class QueryProfileCloneMicroBenchmark { + + private final String description; + private final int propertyCount; + private final String propertyPrefix; + private final boolean useDimensions; + + public QueryProfileCloneMicroBenchmark(String description, int propertyCount, String propertyPrefix, boolean useDimensions) { + this.description=description; + this.propertyCount=propertyCount; + this.propertyPrefix=propertyPrefix; + this.useDimensions=useDimensions; + } + + public void benchmark(int clone) { + cloneQueryWithProfile(10000); // warm-up + System.out.println(description); + long startTime=System.currentTimeMillis(); + cloneQueryWithProfile(clone); + long endTime=System.currentTimeMillis(); + long totalTime=(endTime-startTime); + System.out.println("Done in " + totalTime + " ms (" + ((float)totalTime/clone + " ms per clone)")); + } + + private void cloneQueryWithProfile(int clones) { + QueryProfile main = new QueryProfile("main"); + main.set("a", "value1", (QueryProfileRegistry)null); + main.set("b", "value2", useDimensions ? new String[] {"x1"} : null, null); + main.set("c", "value3", useDimensions ? new String[] {"x1","y2"} : null, null); + main.freeze(); + Query query = new Query(HttpRequest.createTestRequest("?query=test&x=1&y=2", Method.GET), main.compile(null)); + setValues(query); + for (int i=0; i<clones; i++) { + if (i%(clones/100)==0) + System.out.print("."); + query.clone(); + } + } + + private void setValues(Query query) { + for (int i=0; i<propertyCount; i++) { + String thisPrefix=propertyPrefix; + if (thisPrefix==null) + thisPrefix="a"+i+".b"+i+"."; + query.properties().set(thisPrefix + "property" + i,"value" + i); + } + } + + public static void main(String[] args) { + int count=100000; + new QueryProfileCloneMicroBenchmark("Cloning a near-empty query ",0,"",false).benchmark(count); + System.out.println(""); + new QueryProfileCloneMicroBenchmark("Cloning a query with 100 properties in root, no dimensions ",100,"",false).benchmark(count); + System.out.println(""); + new QueryProfileCloneMicroBenchmark("Cloning a query with 100 properties in 1-level nested profiles, no dimensions ",100,"a.",false).benchmark(count); + System.out.println(""); + new QueryProfileCloneMicroBenchmark("Cloning a query with 100 properties in 2-level nested profiles, no dimensions ",100,"a.b.",false).benchmark(count); + System.out.println(""); + new QueryProfileCloneMicroBenchmark("Cloning a query with 100 properties in variable prefix profiles, no dimensions ",100,null,true).benchmark(count); + System.out.println(""); + new QueryProfileCloneMicroBenchmark("Cloning a query with 100 properties in root, with dimensions ",100,"",true).benchmark(count); + System.out.println(""); + new QueryProfileCloneMicroBenchmark("Cloning a query with 100 properties in 1-level nested profiles, with dimensions",100,"a.",true).benchmark(count); + System.out.println(""); + new QueryProfileCloneMicroBenchmark("Cloning a query with 100 properties in 2-level nested profiles, with dimensions",100,"a.b.",true).benchmark(count); + System.out.println(""); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java new file mode 100644 index 00000000000..180c21c54c6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.DimensionValues; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.test.QueryTestCase; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author bratseth + */ +public class QueryProfileGetInComplexStructureMicroBenchmark { + + private final int dotDepth, variantCount, variantParentCount; + + public QueryProfileGetInComplexStructureMicroBenchmark(int dotDepth,int variantCount,int variantParentCount) { + if (dotDepth<0) throw new IllegalArgumentException("dotDepth must be >=0"); + this.dotDepth=dotDepth; + if (variantCount<1) throw new IllegalArgumentException("variantCount must be >0"); + this.variantCount=variantCount; + if (variantParentCount<0) throw new IllegalArgumentException("varientParentCount must be >=0"); + this.variantParentCount=variantParentCount; + } + + public void benchmark(int count, boolean useVariant) { + QueryProfile main=createProfile(useVariant); + Query query = new Query(QueryTestCase.httpEncode("?query=test&x=1&y=2"), main.compile(null)); + getValues(100000,query); // warm-up + System.out.print(this + ": "); + long startTime=System.currentTimeMillis(); + getValues(count,query); + long endTime=System.currentTimeMillis(); + long totalTime=(endTime-startTime); + System.out.println("Done in " + totalTime + " ms (" + ((float) totalTime * 1000 / (count * 2) + " microsecond per get)")); // *2 because we do 2 gets + } + + private QueryProfile createProfile(boolean useVariant) { + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"d0"}); + String prefix=generatePrefix(); + for (int i=0; i<variantCount; i++) + main.set(prefix + "a","value-" + i, new String[] {"dv" + i}, null); + for (int i=0; i<variantParentCount; i++) { + main.addInherited(createParent(i), useVariant ? DimensionValues.createFrom(new String[] {"dv" + i}) : null); + } + main.freeze(); + return main; + } + + private QueryProfile createParent(int i) { + QueryProfile main=new QueryProfile("parent" + i); + main.setDimensions(new String[] {"d0"}); + String prefix=generatePrefix(); + for (int j=0; j<variantCount; j++) + main.set(prefix + "a","value-" + j + "-inherit" + i,new String[] {"dv" + j}, null); + main.freeze(); + return main; + } + + private void getValues(int count,Query query) { + Map<String,String> dimensionValues=createDimensionValueMap(); + String prefix=generatePrefix(); + final int dotInterval=1000000; + final CompoundName found = new CompoundName(prefix + "a"); + final CompoundName notFound = new CompoundName(prefix + "nonexisting"); + for (int i=0; i<count; i++) { + if (count>dotInterval && i%(dotInterval)==0) + System.out.print("."); + if (null==query.properties().get(found,dimensionValues)) // request the last variant for worst case + throw new RuntimeException("Expected value"); + if (null!=query.properties().get(notFound,dimensionValues)) // request the last variant for worst case + throw new RuntimeException("Did not expect value"); + } + } + + private Map<String,String> createDimensionValueMap() { + Map<String,String> dimensionValueMap=new HashMap<>(); + dimensionValueMap.put("d0","dv" + (variantCount-1)); + return dimensionValueMap; + } + + private String generatePrefix() { + StringBuilder b=new StringBuilder(); + for (int i=0; i<dotDepth; i++) + b.append("a."); + return b.toString(); + } + + @Override + public String toString() { + return "dot depth: " + dotDepth + ", variant count: " + variantCount + ", variant parent count: " + variantParentCount; + } + + private static void runBenchmarks(int count, boolean useVariants) { + new QueryProfileGetInComplexStructureMicroBenchmark(1,1,1).benchmark(count, useVariants); + new QueryProfileGetInComplexStructureMicroBenchmark(0,1,0).benchmark(count, useVariants); + + new QueryProfileGetInComplexStructureMicroBenchmark(9,1,0).benchmark(count, useVariants); + new QueryProfileGetInComplexStructureMicroBenchmark(0,9,0).benchmark(count, useVariants); + new QueryProfileGetInComplexStructureMicroBenchmark(9,9,0).benchmark(count, useVariants); + new QueryProfileGetInComplexStructureMicroBenchmark(0,1,9).benchmark(count, useVariants); + new QueryProfileGetInComplexStructureMicroBenchmark(9,1,9).benchmark(count, useVariants); + new QueryProfileGetInComplexStructureMicroBenchmark(0,9,9).benchmark(count, useVariants); + new QueryProfileGetInComplexStructureMicroBenchmark(9,9,9).benchmark(count, useVariants); + } + + public static void main(String[] args) { + System.out.println("Variant benchmarks"); + runBenchmarks(10000000, true); + System.out.println(""); + System.out.println("Inheritance benchmarks"); + runBenchmarks(10000000, false); + System.out.println(""); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java new file mode 100644 index 00000000000..46230793d4d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; + + +/** + * @author bratseth + */ +public class QueryProfileGetMicroBenchmark { + + private final String description; + private final String propertyPrefix; + private final boolean useDimensions; + + public QueryProfileGetMicroBenchmark(String description, String propertyPrefix, boolean useDimensions) { + this.description=description; + this.propertyPrefix=propertyPrefix; + this.useDimensions=useDimensions; + } + + public void benchmark(int count) { + Query query=createQuery(); + getValues(100000,query); // warm-up + System.out.println(description); + long startTime=System.currentTimeMillis(); + getValues(count,query); + long endTime=System.currentTimeMillis(); + long totalTime=(endTime-startTime); + System.out.println("Done in " + totalTime + " ms (" + ((float)totalTime*1000/(count*2) + " microsecond per get)")); // *2 because we do 2 gets + } + + private Query createQuery() { + QueryProfile main = new QueryProfile("main"); + main.set("a", "value1", (QueryProfileRegistry)null); + main.set("b", "value2", useDimensions ? new String[] {"x1"} : null, null); + main.set("c", "value3", useDimensions ? new String[] {"x1","y2"} : null, null); + main.freeze(); + Query query = new Query(HttpRequest.createTestRequest("?query=test&x=1&y=2", Method.GET), main.compile(null)); + setValues(query); + return query; + } + + private void setValues(Query query) { + for (int i=0; i<10; i++) { + String thisPrefix=propertyPrefix; + if (thisPrefix==null) + thisPrefix= "a"+i+".b"+i+"."; + query.properties().set(thisPrefix + "property" + i,"value" + i); + } + } + + private void getValues(int count,Query query) { + final int dotInterval=10000000; + CompoundName found = new CompoundName(propertyPrefix + "property1"); + CompoundName notFound = new CompoundName(propertyPrefix + "nonExisting"); + for (int i=0; i<count; i++) { + if (count>dotInterval && i%(count/dotInterval)==0) + System.out.print("."); + if (null==query.properties().get(found)) + throw new RuntimeException("Expected value"); + if (null!=query.properties().get(notFound)) + throw new RuntimeException("Expected no value"); + } + } + + public static void main(String[] args) { + int count=10000000; + new QueryProfileGetMicroBenchmark("Getting values in root, no dimensions ","",false).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in 1-level nested profiles, no dimensions ","a.",false).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in 2-level nested profiles, no dimensions ","a.b.",false).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in root, with dimensions ","",true).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in 1-level nested profiles, with dimensions","a.",true).benchmark(count); + System.out.println(""); + new QueryProfileGetMicroBenchmark("Getting values in 2-level nested profiles, with dimensions","a.b.",true).benchmark(count); + System.out.println(""); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileListPropertiesMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileListPropertiesMicroBenchmark.java new file mode 100644 index 00000000000..cca4a2833d0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileListPropertiesMicroBenchmark.java @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author bratseth + */ +public class QueryProfileListPropertiesMicroBenchmark { + + private final String description; + private final String propertyPrefix; + private final boolean useDimensions; + + public QueryProfileListPropertiesMicroBenchmark(String description, String propertyPrefix, boolean useDimensions) { + this.description=description; + this.propertyPrefix=propertyPrefix; + this.useDimensions=useDimensions; + } + + public void benchmark(int count) { + Query query=createQuery(); + listValues(10000, query); // warm-up + System.out.println(description); + long startTime=System.currentTimeMillis(); + listValues(count, query); + long endTime=System.currentTimeMillis(); + long totalTime=(endTime-startTime); + System.out.println("Done in " + totalTime + " ms (" + ((float)totalTime*1000/(count) + " microsecond per listProperties)")); + } + + private Query createQuery() { + Map<String,String> dimensions=null; + if (useDimensions) { + dimensions=new HashMap<>(); + dimensions.put("x","1"); + dimensions.put("y","2"); + } + + QueryProfile main=new QueryProfile("main"); + setValues(10,main,dimensions); + QueryProfile parent=new QueryProfile("parent"); + setValues(5,main,dimensions); + main.addInherited(parent); + main.freeze(); + Query query = new Query(HttpRequest.createTestRequest("?query=test&x=1&y=2", Method.GET), main.compile(null)); + return query; + } + + private void setValues(int count,QueryProfile profile,Map<String,String> dimensions) { + for (int i=0; i<count; i++) { + String thisPrefix=propertyPrefix; + if ( ! thisPrefix.isEmpty()) + thisPrefix+="."; + profile.set(thisPrefix + "property" + i, "value" + i, dimensions, null); + } + } + + private void listValues(int count,Query query) { + final int dotInterval=1000000; + for (int i=0; i<count; i++) { + if (count>dotInterval && i%(count/dotInterval)==0) + System.out.print("."); + Map<String,Object> properties = query.properties().listProperties(propertyPrefix); + int expectedSize = 10 + (propertyPrefix.isEmpty() ? 3 : 0); // 3 extra properties on the root + if ( properties.size() != expectedSize ) + throw new RuntimeException("Expected a map of 10 elements, but got " + expectedSize + ": \n" + toString(properties)); + } + } + + private String toString(Map<String,Object> map) { + StringBuilder b=new StringBuilder(); + for (Map.Entry<String,Object> entry : map.entrySet()) + b.append(" ") + .append(entry.getKey()) + .append(" = ") + .append(entry.getValue().toString()) + .append("\n"); + return b.toString(); + } + + public static void main(String[] args) { + int count=1000000; + new QueryProfileListPropertiesMicroBenchmark("Listing values in root, no dimensions ","",false).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in 1-level nested profiles, no dimensions ","a",false).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in 2-level nested profiles, no dimensions ","a.b",false).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in root, with dimensions ","",true).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in 1-level nested profiles, with dimensions","a",true).benchmark(count); + System.out.println(""); + new QueryProfileListPropertiesMicroBenchmark("Listing values in 2-level nested profiles, with dimensions","a.b",true).benchmark(count); + System.out.println(""); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java new file mode 100644 index 00000000000..27f48e3ff6d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileSubstitutionTestCase.java @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.processing.request.Properties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.yolean.Exceptions; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; + +/** + * @author bratseth + */ +public class QueryProfileSubstitutionTestCase extends junit.framework.TestCase { + + public void testSingleSubstitution() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world", "world", (QueryProfileRegistry)null); + assertEquals("Hello world!",p.compile(null).get("message")); + + QueryProfile p2=new QueryProfile("test2"); + p2.addInherited(p); + p2.set("world", "universe", (QueryProfileRegistry)null); + assertEquals("Hello universe!",p2.compile(null).get("message")); + } + + public void testMultipleSubstitutions() { + QueryProfile p=new QueryProfile("test"); + p.set("message","%{greeting} %{entity}%{exclamation}", (QueryProfileRegistry)null); + p.set("greeting","Hola", (QueryProfileRegistry)null); + p.set("entity","local group", (QueryProfileRegistry)null); + p.set("exclamation","?", (QueryProfileRegistry)null); + assertEquals("Hola local group?",p.compile(null).get("message")); + + QueryProfile p2=new QueryProfile("test2"); + p2.addInherited(p); + p2.set("entity","milky way", (QueryProfileRegistry)null); + assertEquals("Hola milky way?",p2.compile(null).get("message")); + } + + public void testUnclosedSubstitution1() { + try { + QueryProfile p=new QueryProfile("test"); + p.set("message1","%{greeting} %{entity}%{exclamation", (QueryProfileRegistry)null); + fail("Should have produced an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'message1' to '%{greeting} %{entity}%{exclamation': Unterminated value substitution '%{exclamation'", + Exceptions.toMessageString(e)); + } + } + + public void testUnclosedSubstitution2() { + try { + QueryProfile p=new QueryProfile("test"); + p.set("message1","%{greeting} %{entity%{exclamation}", (QueryProfileRegistry)null); + fail("Should have produced an exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'message1' to '%{greeting} %{entity%{exclamation}': Unterminated value substitution '%{entity%{exclamation}'", + Exceptions.toMessageString(e)); + } + } + + public void testNullSubstitution() { + QueryProfile p=new QueryProfile("test"); + p.set("message","%{greeting} %{entity}%{exclamation}", (QueryProfileRegistry)null); + p.set("greeting","Hola", (QueryProfileRegistry)null); + assertEquals("Hola ", p.compile(null).get("message")); + + QueryProfile p2=new QueryProfile("test2"); + p2.addInherited(p); + p2.set("greeting","Hola", (QueryProfileRegistry)null); + p2.set("exclamation", "?", (QueryProfileRegistry)null); + assertEquals("Hola ?",p2.compile(null).get("message")); + } + + public void testNoOverridingOfPropertiesSetAtRuntime() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world","world", (QueryProfileRegistry)null); + p.freeze(); + + Properties runtime=new QueryProfileProperties(p.compile(null)); + runtime.set("runtimeMessage","Hello %{world}!"); + assertEquals("Hello world!", runtime.get("message")); + assertEquals("Hello %{world}!",runtime.get("runtimeMessage")); + } + + public void testButPropertiesSetAtRuntimeAreUsedInSubstitutions() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world","world", (QueryProfileRegistry)null); + + Properties runtime=new QueryProfileProperties(p.compile(null)); + runtime.set("world","Earth"); + assertEquals("Hello Earth!",runtime.get("message")); + } + + public void testInspection() { + QueryProfile p=new QueryProfile("test"); + p.set("message", "%{greeting} %{entity}%{exclamation}", (QueryProfileRegistry)null); + assertEquals("message","%{greeting} %{entity}%{exclamation}", + p.declaredContent().entrySet().iterator().next().getValue().toString()); + } + + public void testVariants() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world","world", (QueryProfileRegistry)null); + p.setDimensions(new String[] {"x"}); + p.set("message","Halo %{world}!",new String[] {"x1"}, null); + p.set("world","Europe",new String[] {"x2"}, null); + + CompiledQueryProfile cp = p.compile(null); + assertEquals("Hello world!", cp.get("message", QueryProfileVariantsTestCase.toMap("x=x?"))); + assertEquals("Halo world!", cp.get("message", QueryProfileVariantsTestCase.toMap("x=x1"))); + assertEquals("Hello Europe!", cp.get("message", QueryProfileVariantsTestCase.toMap("x=x2"))); + } + + public void testRecursion() { + QueryProfile p=new QueryProfile("test"); + p.set("message","Hello %{world}!", (QueryProfileRegistry)null); + p.set("world","sol planet number %{number}", (QueryProfileRegistry)null); + p.set("number",3, (QueryProfileRegistry)null); + assertEquals("Hello sol planet number 3!",p.compile(null).get("message")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java new file mode 100644 index 00000000000..ba066367fb0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -0,0 +1,562 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.processing.request.Properties; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +/** + * Tests untyped query profiles + * + * @author bratseth + */ +public class QueryProfileTestCase extends junit.framework.TestCase { + + public void testBasics() { + QueryProfile profile=new QueryProfile("test"); + profile.set("a","a-value", (QueryProfileRegistry)null); + profile.set("b.c","b.c-value", (QueryProfileRegistry)null); + profile.set("d.e.f","d.e.f-value", (QueryProfileRegistry)null); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("a-value",cprofile.get("a")); + assertEquals("b.c-value",cprofile.get("b.c")); + assertEquals("d.e.f-value",cprofile.get("d.e.f")); + + assertNull(cprofile.get("nonexistent")); + assertNull(cprofile.get("nested.nonexistent")); + + assertTrue(profile.lookup("b",null).getClass()==QueryProfile.class); + assertTrue(profile.lookup("b",null).getClass()==QueryProfile.class); + } + + /** Tests cloning, with wrappers used in production in place */ + public void testCloning() { + QueryProfile classProfile=new QueryProfile("test"); + classProfile.set("a","aValue", (QueryProfileRegistry)null); + classProfile.set("b",3, (QueryProfileRegistry)null); + + Properties properties = new QueryProfileProperties(classProfile.compile(null)); + + Properties propertiesClone=properties.clone(); + assertEquals("aValue",propertiesClone.get("a")); + assertEquals(3,propertiesClone.get("b")); + properties.set("a","aNewValue"); + assertEquals("aNewValue",properties.get("a")); + assertEquals("aValue",propertiesClone.get("a")); + } + + public void testFreezing() { + QueryProfile profile=new QueryProfile("test"); + profile.set("a","a-value", (QueryProfileRegistry)null); + profile.set("b.c","b.c-value", (QueryProfileRegistry)null); + profile.set("d.e.f","d.e.f-value", (QueryProfileRegistry)null); + + assertFalse(profile.isFrozen()); + assertEquals("a-value",profile.get("a")); + + profile.freeze(); + + assertTrue(profile.isFrozen()); + assertTrue(((QueryProfile)profile.lookup("b",null)).isFrozen()); + assertTrue(((QueryProfile)profile.lookup("d.e",null)).isFrozen()); + + try { + profile.set("a","value", (QueryProfileRegistry)null); + fail("Expected exception"); + } + catch (IllegalStateException e) { + } + } + + private void assertSameObjects(CompiledQueryProfile profile, String path, List<String> expectedKeys) { + Map<String, Object> subObjects = profile.listValues(path); + assertEquals("Sub-objects list equal for path " + path, new HashSet<>(expectedKeys), subObjects.keySet()); + for(String key : expectedKeys) { + assertEquals("Equal for key " + key, profile.get(key),subObjects.get(path + "." + key)); + } + + } + + public void testGetSubObjects() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + barn.addInherited(mor); + barn.addInherited(far); + mormor.set("a.mormor","a.mormor", (QueryProfileRegistry)null); + barn.set("a.barn","a.barn", (QueryProfileRegistry)null); + mor.set("b.mor", "b.mor", (QueryProfileRegistry)null); + far.set("b.far", "b.far", (QueryProfileRegistry)null); + far.set("a.far","a.far", (QueryProfileRegistry)null); + CompiledQueryProfile cbarn = barn.compile(null); + + assertSameObjects(cbarn, "a", Arrays.asList("mormor","far","barn")); + + assertEquals("b.mor", cbarn.get("b.mor")); + assertEquals("b.far", cbarn.get("b.far")); + } + + public void testInheritance() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + barn.addInherited(mor); + barn.addInherited(far); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + + morfar.set("a","morfar-a", (QueryProfileRegistry)null); + mormor.set("a","mormor-a", (QueryProfileRegistry)null); + farfar.set("a","farfar-a", (QueryProfileRegistry)null); + mor.set("a","mor-a", (QueryProfileRegistry)null); + far.set("a","far-a", (QueryProfileRegistry)null); + barn.set("a","barn-a", (QueryProfileRegistry)null); + + mormor.set("b","mormor-b", (QueryProfileRegistry)null); + far.set("b","far-b", (QueryProfileRegistry)null); + + mor.set("c","mor-c", (QueryProfileRegistry)null); + far.set("c","far-c", (QueryProfileRegistry)null); + + mor.set("d.a","mor-d.a", (QueryProfileRegistry)null); + barn.set("d.b","barn-d.b", (QueryProfileRegistry)null); + + QueryProfile annetBarn=new QueryProfile("annetBarn"); + annetBarn.set("venn",barn, (QueryProfileRegistry)null); + + CompiledQueryProfile cbarn = barn.compile(null); + CompiledQueryProfile cannetBarn = annetBarn.compile(null); + + assertEquals("barn-a", cbarn.get("a")); + assertEquals("mormor-b", cbarn.get("b")); + assertEquals("mor-c", cbarn.get("c")); + + assertEquals("barn-a", cannetBarn.get("venn.a")); + assertEquals("mormor-b", cannetBarn.get("venn.b")); + assertEquals("mor-c", cannetBarn.get("venn.c")); + + assertEquals("barn-d.b", cbarn.get("d.b")); + assertEquals("mor-d.a", cbarn.get("d.a")); + } + + public void testInheritance2Level() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + barn.addInherited(mor); + barn.addInherited(far); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + + morfar.set("a.x","morfar-a", (QueryProfileRegistry)null); + mormor.set("a.x","mormor-a", (QueryProfileRegistry)null); + farfar.set("a.x","farfar-a", (QueryProfileRegistry)null); + mor.set("a.x","mor-a", (QueryProfileRegistry)null); + far.set("a.x","far-a", (QueryProfileRegistry)null); + barn.set("a.x","barn-a", (QueryProfileRegistry)null); + + mormor.set("b.x","mormor-b", (QueryProfileRegistry)null); + far.set("b.x","far-b", (QueryProfileRegistry)null); + + mor.set("c.x","mor-c", (QueryProfileRegistry)null); + far.set("c.x","far-c", (QueryProfileRegistry)null); + + mor.set("d.a.x","mor-d.a", (QueryProfileRegistry)null); + barn.set("d.b.x","barn-d.b", (QueryProfileRegistry)null); + + QueryProfile annetBarn=new QueryProfile("annetBarn"); + annetBarn.set("venn",barn, (QueryProfileRegistry)null); + + CompiledQueryProfile cbarn = barn.compile(null); + CompiledQueryProfile cannetBarn = annetBarn.compile(null); + + assertEquals("barn-a", cbarn.get("a.x")); + assertEquals("mormor-b", cbarn.get("b.x")); + assertEquals("mor-c", cbarn.get("c.x")); + + assertEquals("barn-a", cannetBarn.get("venn.a.x")); + assertEquals("mormor-b", cannetBarn.get("venn.b.x")); + assertEquals("mor-c", cannetBarn.get("venn.c.x")); + + assertEquals("barn-d.b", cbarn.get("d.b.x")); + assertEquals("mor-d.a", cbarn.get("d.a.x")); + } + + public void testInheritance3Level() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + barn.addInherited(mor); + barn.addInherited(far); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + + morfar.set("y.a.x","morfar-a", (QueryProfileRegistry)null); + mormor.set("y.a.x","mormor-a", (QueryProfileRegistry)null); + farfar.set("y.a.x","farfar-a", (QueryProfileRegistry)null); + mor.set("y.a.x","mor-a", (QueryProfileRegistry)null); + far.set("y.a.x","far-a", (QueryProfileRegistry)null); + barn.set("y.a.x","barn-a", (QueryProfileRegistry)null); + + mormor.set("y.b.x","mormor-b", (QueryProfileRegistry)null); + far.set("y.b.x","far-b", (QueryProfileRegistry)null); + + mor.set("y.c.x","mor-c", (QueryProfileRegistry)null); + far.set("y.c.x","far-c", (QueryProfileRegistry)null); + + mor.set("y.d.a.x","mor-d.a", (QueryProfileRegistry)null); + barn.set("y.d.b.x","barn-d.b", (QueryProfileRegistry)null); + + QueryProfile annetBarn=new QueryProfile("annetBarn"); + annetBarn.set("venn",barn, (QueryProfileRegistry)null); + + CompiledQueryProfile cbarn = barn.compile(null); + CompiledQueryProfile cannetBarn = annetBarn.compile(null); + + assertEquals("barn-a", cbarn.get("y.a.x")); + assertEquals("mormor-b", cbarn.get("y.b.x")); + assertEquals("mor-c", cbarn.get("y.c.x")); + + assertEquals("barn-a", cannetBarn.get("venn.y.a.x")); + assertEquals("mormor-b", cannetBarn.get("venn.y.b.x")); + assertEquals("mor-c", cannetBarn.get("venn.y.c.x")); + + assertEquals("barn-d.b", cbarn.get("y.d.b.x")); + assertEquals("mor-d.a", cbarn.get("y.d.a.x")); + } + + public void testListProperties() { + QueryProfile barn=new QueryProfile("barn"); + QueryProfile mor=new QueryProfile("mor"); + QueryProfile far=new QueryProfile("far"); + QueryProfile mormor=new QueryProfile("mormor"); + QueryProfile morfar=new QueryProfile("morfar"); + QueryProfile farfar=new QueryProfile("farfar"); + barn.addInherited(mor); + barn.addInherited(far); + mor.addInherited(mormor); + mor.addInherited(morfar); + far.addInherited(farfar); + + morfar.set("a","morfar-a", (QueryProfileRegistry)null); + morfar.set("model.b","morfar-model.b", (QueryProfileRegistry)null); + mormor.set("a","mormor-a", (QueryProfileRegistry)null); + mormor.set("model.b","mormor-model.b", (QueryProfileRegistry)null); + farfar.set("a","farfar-a", (QueryProfileRegistry)null); + mor.set("a","mor-a", (QueryProfileRegistry)null); + far.set("a","far-a", (QueryProfileRegistry)null); + barn.set("a","barn-a", (QueryProfileRegistry)null); + mormor.set("b","mormor-b", (QueryProfileRegistry)null); + far.set("b","far-b", (QueryProfileRegistry)null); + mor.set("c","mor-c", (QueryProfileRegistry)null); + far.set("c","far-c", (QueryProfileRegistry)null); + + CompiledQueryProfile cbarn = barn.compile(null); + + QueryProfileProperties properties = new QueryProfileProperties(cbarn); + + assertEquals("barn-a", cbarn.get("a")); + assertEquals("mormor-b", cbarn.get("b")); + + Map<String, Object> rootMap = properties.listProperties(); + assertEquals("barn-a", rootMap.get("a")); + assertEquals("mormor-b", rootMap.get("b")); + assertEquals("mor-c", rootMap.get("c")); + + Map<String, Object> modelMap = properties.listProperties("model"); + assertEquals("mormor-model.b", modelMap.get("b")); + + QueryProfile annetBarn=new QueryProfile("annetBarn"); + annetBarn.set("venn", barn, (QueryProfileRegistry)null); + CompiledQueryProfile cannetBarn = annetBarn.compile(null); + + Map<String, Object> annetBarnMap = new QueryProfileProperties(cannetBarn).listProperties(); + assertEquals("barn-a", annetBarnMap.get("venn.a")); + assertEquals("mormor-b", annetBarnMap.get("venn.b")); + assertEquals("mor-c", annetBarnMap.get("venn.c")); + assertEquals("mormor-model.b", annetBarnMap.get("venn.model.b")); + } + + /** Tests that dots are followed when setting overridability */ + public void testInstanceOverridable() { + QueryProfile profile=new QueryProfile("root/unoverridableIndex"); + profile.set("model.defaultIndex","default", (QueryProfileRegistry)null); + profile.setOverridable("model.defaultIndex",false,null); + + assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null).booleanValue()); + + // Parameters should be ignored + Query query = new Query(HttpRequest.createTestRequest("?model.defaultIndex=title", Method.GET), profile.compile(null)); + assertEquals("default",query.getModel().getDefaultIndex()); + + // Parameters should be ignored + query = new Query(HttpRequest.createTestRequest("?model.defaultIndex=title&model.language=de", Method.GET), profile.compile(null)); + assertEquals("default",query.getModel().getDefaultIndex()); + assertEquals("de",query.getModel().getLanguage().languageCode()); + } + + /** Tests that dots are followed when setting overridability...also with variants */ + public void testInstanceOverridableWithVariants() { + QueryProfile profile=new QueryProfile("root/unoverridableIndex"); + profile.setDimensions(new String[] {"x"}); + profile.set("model.defaultIndex","default", (QueryProfileRegistry)null); + profile.setOverridable("model.defaultIndex",false,null); + + assertFalse(profile.isDeclaredOverridable("model.defaultIndex",null)); + + // Parameters should be ignored + Query query = new Query(HttpRequest.createTestRequest("?x=x1&model.defaultIndex=title", Method.GET), profile.compile(null)); + assertEquals("default",query.getModel().getDefaultIndex()); + + // Parameters should be ignored + query = new Query(HttpRequest.createTestRequest("?x=x1&model.default-index=title&model.language=de", Method.GET), profile.compile(null)); + assertEquals("default",query.getModel().getDefaultIndex()); + assertEquals("de",query.getModel().getLanguage().languageCode()); + } + + public void testSimpleInstanceOverridableWithVariants1() { + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a","original", (QueryProfileRegistry)null); + profile.setOverridable("a",false,null); + + assertFalse(profile.isDeclaredOverridable("a",null)); + + Query query = new Query(HttpRequest.createTestRequest("?x=x1&a=overridden", Method.GET), profile.compile(null)); + assertEquals("original",query.properties().get("a")); + } + + public void testSimpleInstanceOverridableWithVariants2() { + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a","original",new String[] {"x1"}, null); + profile.setOverridable("a",false,null); + + assertFalse(profile.isDeclaredOverridable("a",null)); + + Query query = new Query(HttpRequest.createTestRequest("?x=x1&a=overridden", Method.GET), profile.compile(null)); + assertEquals("original",query.properties().get("a")); + } + + /** Tests having both an explicit reference and an override */ + public void testExplicitReferenceOverride() { + QueryProfile a1=new QueryProfile("a1"); + a1.set("b","a1.b", (QueryProfileRegistry)null); + QueryProfile profile=new QueryProfile("test"); + profile.set("a",a1, (QueryProfileRegistry)null); + profile.set("a.b","a.b", (QueryProfileRegistry)null); + assertEquals("a.b",profile.compile(null).get("a.b")); + } + + public void testSettingNonLeaf1() { + QueryProfile p=new QueryProfile("test"); + p.set("a","a-value", (QueryProfileRegistry)null); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + assertEquals("a-value", cp.get("a")); + assertEquals("a.b-value", cp.get("a.b")); + } + + public void testSettingNonLeaf2() { + QueryProfile p=new QueryProfile("test"); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + p.set("a","a-value", (QueryProfileRegistry)null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + assertEquals("a-value", cp.get("a")); + assertEquals("a.b-value", cp.get("a.b")); + } + + public void testSettingNonLeaf3a() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + p.set("a","a-value",new String[] {"x1"}, null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertNull(p.get("a")); + assertEquals("a.b-value", cp.get("a.b")); + assertEquals("a-value", cp.get("a", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + assertEquals("a.b-value", cp.get("a.b", new String[] {"x1"})); + } + + public void testSettingNonLeaf3b() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a","a-value",new String[] {"x1"}, null); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertNull(cp.get("a")); + assertEquals("a.b-value", cp.get("a.b")); + assertEquals("a-value", cp.get("a", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + assertEquals("a.b-value", cp.get("a.b",new String[] {"x1"})); + } + + public void testSettingNonLeaf4a() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a.b","a.b-value",new String[] {"x1"}, null); + p.set("a","a-value", (QueryProfileRegistry)null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertEquals("a-value", cp.get("a")); + assertNull(cp.get("a.b")); + assertEquals("a-value", cp.get("a",new String[] {"x1"})); + assertEquals("a.b-value", cp.get("a.b", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + } + + public void testSettingNonLeaf4b() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a","a-value", (QueryProfileRegistry)null); + p.set("a.b","a.b-value",new String[] {"x1"}, null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertEquals("a-value", cp.get("a")); + assertNull(cp.get("a.b")); + assertEquals("a-value", cp.get("a",new String[] {"x1"})); + assertEquals("a.b-value", cp.get("a.b", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + } + + public void testSettingNonLeaf5() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a.b","a.b-value",new String[] {"x1"}, null); + p.set("a","a-value",new String[] {"x1"}, null); + + QueryProfileProperties cp = new QueryProfileProperties(p.compile(null)); + + assertNull(cp.get("a")); + assertNull(cp.get("a.b")); + assertEquals("a-value", cp.get("a", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + assertEquals("a.b-value", cp.get("a.b", QueryProfileVariantsTestCase.toMap(p, new String[] {"x1"}))); + } + + public void testListingWithNonLeafs() { + QueryProfile p=new QueryProfile("test"); + p.set("a","a-value", (QueryProfileRegistry)null); + p.set("a.b","a.b-value", (QueryProfileRegistry)null); + Map<String,Object> values = p.compile(null).listValues("a"); + assertEquals(1,values.size()); + assertEquals("a.b-value",values.get("b")); + } + + public void testRankTypeNames() { + QueryProfile p=new QueryProfile("test"); + p.set("a.$b","foo", (QueryProfileRegistry)null); + p.set("a.query(b)","bar", (QueryProfileRegistry)null); + p.set("a.b.default-index","fuu", (QueryProfileRegistry)null); + CompiledQueryProfile cp = p.compile(null); + + assertEquals("foo", cp.get("a.$b")); + assertEquals("bar", cp.get("a.query(b)")); + assertEquals("fuu", cp.get("a.b.default-index")); + + Map<String,Object> p1 = cp.listValues(""); + assertEquals("foo", p1.get("a.$b")); + assertEquals("bar", p1.get("a.query(b)")); + assertEquals("fuu", p1.get("a.b.default-index")); + + Map<String,Object> p2 = cp.listValues("a"); + assertEquals("foo", p2.get("$b")); + assertEquals("bar", p2.get("query(b)")); + assertEquals("fuu", p2.get("b.default-index")); + } + + public void testQueryProfileInlineValueReassignment() { + QueryProfile p=new QueryProfile("test"); + p.set("source.rel.params.query","%{model.queryString}", (QueryProfileRegistry)null); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?query=foo", Method.GET), p.compile(null)); + assertEquals("foo",q.properties().get("source.rel.params.query")); + assertEquals("foo",q.properties().listProperties().get("source.rel.params.query")); + q.getModel().setQueryString("bar"); + assertEquals("bar",q.properties().get("source.rel.params.query")); + assertEquals("foo",q.properties().listProperties().get("source.rel.params.query")); // Is still foo because model variables are not supported with the list function + } + + public void testQueryProfileInlineValueReassignmentSimpleName() { + QueryProfile p=new QueryProfile("test"); + p.set("key","%{model.queryString}", (QueryProfileRegistry)null); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?query=foo", Method.GET), p.compile(null)); + assertEquals("foo",q.properties().get("key")); + assertEquals("foo",q.properties().listProperties().get("key")); + q.getModel().setQueryString("bar"); + assertEquals("bar",q.properties().get("key")); + assertEquals("foo",q.properties().listProperties().get("key")); // Is still bar because model variables are not supported with the list function + } + + public void testQueryProfileInlineValueReassignmentSimpleNameGenericProperty() { + QueryProfile p=new QueryProfile("test"); + p.set("key","%{value}", (QueryProfileRegistry)null); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?query=test&value=foo", Method.GET), p.compile(null)); + assertEquals("foo",q.properties().get("key")); + assertEquals("foo",q.properties().listProperties().get("key")); + q.properties().set("value","bar"); + assertEquals("bar",q.properties().get("key")); + assertEquals("bar",q.properties().listProperties().get("key")); + } + + public void testQueryProfileModelValueListing() { + QueryProfile p=new QueryProfile("test"); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?query=bar", Method.GET), p.compile(null)); + assertEquals("bar",q.properties().get("model.queryString")); + assertEquals("bar",q.properties().listProperties().get("model.queryString")); + q.getModel().setQueryString("baz"); + assertEquals("baz",q.properties().get("model.queryString")); + assertEquals("bar",q.properties().listProperties().get("model.queryString")); // Is still bar because model variables are not supported with the list function + } + + public void testEmptyBoolean() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x","y"}); + p.set("clustering.something","bar", (QueryProfileRegistry)null); + p.set("clustering.something","bar",new String[] {"x1","y1"}, null); + p.freeze(); + Query q = new Query(HttpRequest.createTestRequest("?x=x1&y=y1&query=bar&clustering.timeline.kano=tur&" + + "clustering.enable=true&clustering.timeline.bucketspec=-" + + "7d/3h&clustering.timeline.tophit=false&clustering.timeli" + + "ne=true", Method.GET),p.compile(null)); + assertEquals(true,q.properties().getBoolean("clustering.timeline",false)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsCloneTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsCloneTestCase.java new file mode 100644 index 00000000000..de8f9dab985 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsCloneTestCase.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + + +import com.yahoo.search.query.profile.DimensionValues; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author tonytv + */ +public class QueryProfileVariantsCloneTestCase { + + /** + * Test for Ticket 4882480. + */ + @Test + public void test_that_interior_and_leaf_values_on_a_path_are_preserved_when_cloning() { + Map<String, String> dimensionBinding = createDimensionBinding("location", "norway"); + + QueryProfile profile = new QueryProfile("profile"); + profile.setDimensions(keys(dimensionBinding)); + + DimensionValues dimensionValues = DimensionValues.createFrom(values(dimensionBinding)); + profile.set("interior.leaf", "leafValue", dimensionValues, null); + profile.set("interior", "interiorValue", dimensionValues, null); + + CompiledQueryProfile clone = profile.compile(null).clone(); + + assertEquals(profile.get("interior", dimensionBinding, null), + clone.get("interior", dimensionBinding)); + + assertEquals(profile.get("interior.leaf", dimensionBinding, null), + clone.get("interior.leaf", dimensionBinding)); + } + + + private static Map<String,String> createDimensionBinding(String dimension, String value) { + Map<String, String> dimensionBinding = new HashMap<>(); + dimensionBinding.put(dimension, value); + return Collections.unmodifiableMap(dimensionBinding); + } + + private static String[] keys(Map<String, String> map) { + return map.keySet().toArray(new String[0]); + } + + private static String[] values(Map<String, String> map) { + return map.values().toArray(new String[0]); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java new file mode 100644 index 00000000000..95f121adab9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java @@ -0,0 +1,1052 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.Query; +import com.yahoo.search.query.Properties; +import com.yahoo.search.query.profile.BackedOverridableQueryProfile; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author bratseth + */ +public class QueryProfileVariantsTestCase extends junit.framework.TestCase { + + public void testSimple() { + QueryProfile profile=new QueryProfile("a"); + profile.set("a","a.deflt", (QueryProfileRegistry)null); + profile.setDimensions(new String[] {"x","y","z"}); + profile.set("a","a.1.*.*",new String[] {"x1",null,null}, null); + profile.set("a","a.1.*.1",new String[] {"x1",null,"z1"}, null); + profile.set("a","a.1.*.5",new String[] {"x1",null,"z5"}, null); + profile.set("a","a.1.1.*",new String[] {"x1","y1",null}, null); + profile.set("a","a.1.5.*",new String[] {"x1","y5",null}, null); + profile.set("a","a.1.1.1",new String[] {"x1","y1","z1"}, null); + profile.set("a","a.2.1.1",new String[] {"x2","y1","z1"}, null); + profile.set("a","a.1.2.2",new String[] {"x1","y2","z2"}, null); + profile.set("a","a.1.2.3",new String[] {"x1","y2","z3"}, null); + profile.set("a","a.2.*.*",new String[] {"x2" }, null); // Same as ,null,null + CompiledQueryProfile cprofile = profile.compile(null); + + // Perfect matches + assertGet("a.deflt","a",new String[] {null,null,null}, profile, cprofile); + assertGet("a.1.*.*","a",new String[] {"x1",null,null}, profile, cprofile); + assertGet("a.1.1.*","a",new String[] {"x1","y1",null}, profile, cprofile); + assertGet("a.1.5.*","a",new String[] {"x1","y5",null}, profile, cprofile); + assertGet("a.1.*.1","a",new String[] {"x1",null,"z1"}, profile, cprofile); + assertGet("a.1.*.5","a",new String[] {"x1",null,"z5"}, profile, cprofile); + assertGet("a.1.1.1","a",new String[] {"x1","y1","z1"}, profile, cprofile); + assertGet("a.2.1.1","a",new String[] {"x2","y1","z1"}, profile, cprofile); + assertGet("a.1.2.2","a",new String[] {"x1","y2","z2"}, profile, cprofile); + assertGet("a.1.2.3","a",new String[] {"x1","y2","z3"}, profile, cprofile); + assertGet("a.2.*.*","a",new String[] {"x2",null,null}, profile, cprofile); + + // Wildcard matches + assertGet("a.deflt","a",new String[] {"x?","y?","z?"}, profile, cprofile); + assertGet("a.deflt","a",new String[] {"x?","y1","z1"}, profile, cprofile); + assertGet("a.1.*.*","a",new String[] {"x1","y?","z?"}, profile, cprofile); + assertGet("a.1.*.*","a",new String[] {"x1","y?","z?"}, profile, cprofile); + assertGet("a.1.1.*","a",new String[] {"x1","y1","z?"}, profile, cprofile); + assertGet("a.1.*.1","a",new String[] {"x1","y?","z1"}, profile, cprofile); + assertGet("a.1.5.*","a",new String[] {"x1","y5","z?"}, profile, cprofile); + assertGet("a.1.*.5","a",new String[] {"x1","y?","z5"}, profile, cprofile); + assertGet("a.1.5.*","a",new String[] {"x1","y5","z5"}, profile, cprofile); // Left dimension gets precedence + assertGet("a.2.*.*","a",new String[] {"x2","y?","z?"}, profile, cprofile); + } + + public void testVariantsOfInlineCompound() { + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a.b","a.b", (QueryProfileRegistry)null); + profile.set("a.b","a.b.x1",new String[] {"x1"}, null); + profile.set("a.b","a.b.x2",new String[] {"x2"}, null); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("a.b",cprofile.get("a.b")); + assertEquals("a.b.x1",cprofile.get("a.b", toMap("x=x1"))); + assertEquals("a.b.x2",cprofile.get("a.b", toMap("x=x2"))); + } + + public void testVariantsOfExplicitCompound() { + QueryProfile a1=new QueryProfile("a1"); + a1.set("b","a.b", (QueryProfileRegistry)null); + + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x"}); + profile.set("a",a1, (QueryProfileRegistry)null); + profile.set("a.b","a.b.x1",new String[] {"x1"}, null); + profile.set("a.b","a.b.x2",new String[] {"x2"}, null); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("a.b",cprofile.get("a.b")); + assertEquals("a.b.x1",cprofile.get("a.b", toMap("x=x1"))); + assertEquals("a.b.x2",cprofile.get("a.b", toMap("x=x2"))); + } + + public void testCompound() { + // Configuration phase + + QueryProfile profile=new QueryProfile("test"); + profile.setDimensions(new String[] {"x","y"}); + + QueryProfile a1=new QueryProfile("a1"); + a1.set("b","a1.b.default", (QueryProfileRegistry)null); + a1.set("c","a1.c.default", (QueryProfileRegistry)null); + a1.set("d","a1.d.default", (QueryProfileRegistry)null); + a1.set("e","a1.e.default", (QueryProfileRegistry)null); + + QueryProfile a2=new QueryProfile("a2"); + a2.set("b","a2.b.default", (QueryProfileRegistry)null); + a2.set("c","a2.c.default", (QueryProfileRegistry)null); + a2.set("d","a2.d.default", (QueryProfileRegistry)null); + a2.set("e","a2.e.default", (QueryProfileRegistry)null); + + profile.set("a",a1, (QueryProfileRegistry)null); // Must set profile references before overrides + profile.set("a.b","a.b.default-override", (QueryProfileRegistry)null); + profile.set("a.c","a.c.default-override", (QueryProfileRegistry)null); + profile.set("a.d","a.d.default-override", (QueryProfileRegistry)null); + profile.set("a.g","a.g.default-override", (QueryProfileRegistry)null); + + String[] d1=new String[] { "x1","y1" }; + profile.set("a",a1,d1, null); + profile.set("a.b","x1.y1.a.b.default-override",d1, null); + profile.set("a.c","x1.y1.a.c.default-override",d1, null); + profile.set("a.g","x1.y1.a.g.default-override",d1, null); // This value is never manifest because the runtime override overrides all variants + + String[] d2=new String[] { "x1","y2" }; + profile.set("a.b","x1.y2.a.b.default-override",d2, null); + profile.set("a.c","x1.y2.a.c.default-override",d2, null); + + String[] d3=new String[] { "x2","y1" }; + profile.set("a",a2,d3, null); + profile.set("a.b","x2.y1.a.b.default-override",d3, null); + profile.set("a.c","x2.y1.a.c.default-override",d3, null); + + + // Runtime phase - four simultaneous requests using different variants makes their own overrides + QueryProfileProperties defaultRuntimeProfile = new QueryProfileProperties(profile.compile(null)); + defaultRuntimeProfile.set("a.f", "a.f.runtime-override"); + defaultRuntimeProfile.set("a.g", "a.g.runtime-override"); + + QueryProfileProperties d1RuntimeProfile = new QueryProfileProperties(profile.compile(null)); + d1RuntimeProfile.set("a.f", "a.f.d1.runtime-override", toMap("x=x1", "y=y1")); + d1RuntimeProfile.set("a.g", "a.g.d1.runtime-override", toMap("x=x1", "y=y1")); + + QueryProfileProperties d2RuntimeProfile = new QueryProfileProperties(profile.compile(null)); + d2RuntimeProfile.set("a.f", "a.f.d2.runtime-override",toMap("x=x1", "y=y2")); + d2RuntimeProfile.set("a.g", "a.g.d2.runtime-override",toMap("x=x1", "y=y2")); + + QueryProfileProperties d3RuntimeProfile = new QueryProfileProperties(profile.compile(null)); + d3RuntimeProfile.set("a.f", "a.f.d3.runtime-override", toMap("x=x2", "y=y1")); + d3RuntimeProfile.set("a.g", "a.g.d3.runtime-override", toMap("x=x2", "y=y1")); + + // Lookups + assertEquals("a.b.default-override", defaultRuntimeProfile.get("a.b")); + assertEquals("a.c.default-override", defaultRuntimeProfile.get("a.c")); + assertEquals("a.d.default-override", defaultRuntimeProfile.get("a.d")); + assertEquals("a1.e.default", defaultRuntimeProfile.get("a.e")); + assertEquals("a.f.runtime-override", defaultRuntimeProfile.get("a.f")); + assertEquals("a.g.runtime-override", defaultRuntimeProfile.get("a.g")); + + assertEquals("x1.y1.a.b.default-override", d1RuntimeProfile.get("a.b", toMap("x=x1", "y=y1"))); + assertEquals("x1.y1.a.c.default-override", d1RuntimeProfile.get("a.c", toMap("x=x1", "y=y1"))); + assertEquals("a1.d.default", d1RuntimeProfile.get("a.d", toMap("x=x1", "y=y1"))); + assertEquals("a1.e.default", d1RuntimeProfile.get("a.e", toMap("x=x1", "y=y1"))); + assertEquals("a.f.d1.runtime-override", d1RuntimeProfile.get("a.f", toMap("x=x1", "y=y1"))); + assertEquals("a.g.d1.runtime-override", d1RuntimeProfile.get("a.g", toMap("x=x1", "y=y1"))); + + assertEquals("x1.y2.a.b.default-override", d2RuntimeProfile.get("a.b", toMap("x=x1", "y=y2"))); + assertEquals("x1.y2.a.c.default-override", d2RuntimeProfile.get("a.c", toMap("x=x1", "y=y2"))); + assertEquals("a.d.default-override", d2RuntimeProfile.get("a.d", toMap("x=x1", "y=y2"))); // Because this variant does not itself refer to a + assertEquals("a1.e.default", d2RuntimeProfile.get("a.e", toMap("x=x1", "y=y2"))); + assertEquals("a.f.d2.runtime-override", d2RuntimeProfile.get("a.f", toMap("x=x1", "y=y2"))); + assertEquals("a.g.d2.runtime-override", d2RuntimeProfile.get("a.g", toMap("x=x1", "y=y2"))); + + assertEquals("x2.y1.a.b.default-override", d3RuntimeProfile.get("a.b", toMap("x=x2", "y=y1"))); + assertEquals("x2.y1.a.c.default-override", d3RuntimeProfile.get("a.c", toMap("x=x2", "y=y1"))); + assertEquals("a2.d.default", d3RuntimeProfile.get("a.d", toMap("x=x2", "y=y1"))); + assertEquals("a2.e.default", d3RuntimeProfile.get("a.e", toMap("x=x2", "y=y1"))); + assertEquals("a.f.d3.runtime-override", d3RuntimeProfile.get("a.f", toMap("x=x2", "y=y1"))); + assertEquals("a.g.d3.runtime-override", d3RuntimeProfile.get("a.g", toMap("x=x2", "y=y1"))); + } + + public void testVariantNotInBase() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + test.set("InX1Only","x1",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("x1",ctest.get("InX1Only", toMap("x=x1"))); + assertEquals(null,ctest.get("InX1Only", toMap("x=x2"))); + assertEquals(null,ctest.get("InX1Only")); + } + + public void testVariantNotInBaseSpaceVariantValue() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + test.set("InX1Only","x1",new String[] {"x 1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + + assertEquals("x1",ctest.get("InX1Only", toMap("x=x 1"))); + assertEquals(null,ctest.get("InX1Only", toMap("x=x 2"))); + assertEquals(null,ctest.get("InX1Only")); + } + + public void testDimensionsInSuperType() { + QueryProfile parent=new QueryProfile("parent"); + parent.setDimensions(new String[] {"x","y"}); + QueryProfile child=new QueryProfile("child"); + child.addInherited(parent); + child.set("a","a.default", (QueryProfileRegistry)null); + child.set("a","a.x1.y1",new String[] {"x1","y1"}, null); + child.set("a","a.x1.y2",new String[] {"x1","y2"}, null); + + CompiledQueryProfile cchild = child.compile(null); + + assertEquals("a.default",cchild.get("a")); + assertEquals("a.x1.y1",cchild.get("a", toMap("x=x1","y=y1"))); + assertEquals("a.x1.y2",cchild.get("a", toMap("x=x1","y=y2"))); + } + + public void testDimensionsInSuperTypeRuntime() { + QueryProfile parent=new QueryProfile("parent"); + parent.setDimensions(new String[] {"x","y"}); + QueryProfile child=new QueryProfile("child"); + child.addInherited(parent); + child.set("a","a.default", (QueryProfileRegistry)null); + child.set("a", "a.x1.y1", new String[]{"x1", "y1"}, null); + child.set("a", "a.x1.y2", new String[]{"x1", "y2"}, null); + Properties overridable=new QueryProfileProperties(child.compile(null)); + + assertEquals("a.default", child.get("a")); + assertEquals("a.x1.y1", overridable.get("a", toMap("x=x1", "y=y1"))); + assertEquals("a.x1.y2", overridable.get("a", toMap("x=x1", "y=y2"))); + } + + public void testVariantsAreResolvedBeforeInheritance() { + QueryProfile parent=new QueryProfile("parent"); + parent.setDimensions(new String[] {"x","y"}); + parent.set("a","p.a.default", (QueryProfileRegistry)null); + parent.set("a","p.a.x1.y1",new String[] {"x1","y1"}, null); + parent.set("a","p.a.x1.y2",new String[] {"x1","y2"}, null); + parent.set("b","p.b.default", (QueryProfileRegistry)null); + parent.set("b","p.b.x1.y1",new String[] {"x1","y1"}, null); + parent.set("b","p.b.x1.y2",new String[] {"x1","y2"}, null); + QueryProfile child=new QueryProfile("child"); + child.setDimensions(new String[] {"x","y"}); + child.addInherited(parent); + child.set("a","c.a.default", (QueryProfileRegistry)null); + child.set("a","c.a.x1.y1",new String[] {"x1","y1"}, null); + + CompiledQueryProfile cchild = child.compile(null); + assertEquals("c.a.default",cchild.get("a")); + assertEquals("c.a.x1.y1",cchild.get("a", toMap("x=x1", "y=y1"))); + assertEquals("c.a.default",cchild.get("a", toMap("x=x1", "y=y2"))); + assertEquals("p.b.default",cchild.get("b")); + assertEquals("p.b.x1.y1",cchild.get("b", toMap("x=x1", "y=y1"))); + assertEquals("p.b.x1.y2",cchild.get("b", toMap("x=x1", "y=y2"))); + } + + public void testVariantsAreResolvedBeforeInheritanceSimplified() { + QueryProfile parent=new QueryProfile("parent"); + parent.setDimensions(new String[] {"x","y"}); + parent.set("a","p.a.x1.y2",new String[] {"x1","y2"}, null); + + QueryProfile child=new QueryProfile("child"); + child.setDimensions(new String[] {"x","y"}); + child.addInherited(parent); + child.set("a","c.a.default", (QueryProfileRegistry)null); + + assertEquals("c.a.default",child.compile(null).get("a", toMap("x=x1", "y=y2"))); + } + + public void testVariantInheritance() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x","y"}); + QueryProfile defaultParent=new QueryProfile("defaultParent"); + defaultParent.set("a","a-default", (QueryProfileRegistry)null); + QueryProfile x1Parent=new QueryProfile("x1Parent"); + x1Parent.set("a","a-x1", (QueryProfileRegistry)null); + x1Parent.set("d","d-x1", (QueryProfileRegistry)null); + x1Parent.set("e","e-x1", (QueryProfileRegistry)null); + QueryProfile x1y1Parent=new QueryProfile("x1y1Parent"); + x1y1Parent.set("a","a-x1y1", (QueryProfileRegistry)null); + QueryProfile x1y2Parent=new QueryProfile("x1y2Parent"); + x1y2Parent.set("a","a-x1y2", (QueryProfileRegistry)null); + x1y2Parent.set("b","b-x1y2", (QueryProfileRegistry)null); + x1y2Parent.set("c","c-x1y2", (QueryProfileRegistry)null); + test.addInherited(defaultParent); + test.addInherited(x1Parent,new String[] {"x1"}); + test.addInherited(x1y1Parent,new String[] {"x1","y1"}); + test.addInherited(x1y2Parent,new String[] {"x1","y2"}); + test.set("c","c-x1",new String[] {"x1"}, null); + test.set("e","e-x1y2",new String[] {"x1","y2"}, null); + + CompiledQueryProfile ctest = test.compile(null); + + assertEquals("a-default",ctest.get("a")); + assertEquals("a-x1",ctest.get("a", toMap("x=x1"))); + assertEquals("a-x1y1",ctest.get("a", toMap("x=x1", "y=y1"))); + assertEquals("a-x1y2",ctest.get("a", toMap("x=x1", "y=y2"))); + + assertEquals(null,ctest.get("b")); + assertEquals(null,ctest.get("b", toMap("x=x1"))); + assertEquals(null,ctest.get("b", toMap("x=x1", "y=y1"))); + assertEquals("b-x1y2",ctest.get("b", toMap("x=x1", "y=y2"))); + + assertEquals(null,ctest.get("c")); + assertEquals("c-x1",ctest.get("c", toMap("x=x1"))); + assertEquals("c-x1",ctest.get("c", toMap("x=x1", "y=y1"))); + assertEquals("c-x1y2",ctest.get("c", toMap("x=x1", "y=y2"))); + + assertEquals(null,ctest.get("d")); + assertEquals("d-x1",ctest.get("d", toMap("x=x1"))); + + assertEquals("d-x1",ctest.get("d", toMap("x=x1", "y=y1"))); + assertEquals("d-x1",ctest.get("d", toMap("x=x1", "y=y2"))); + + assertEquals(null,ctest.get("d")); + assertEquals("e-x1",ctest.get("e", toMap("x=x1"))); + assertEquals("e-x1",ctest.get("e", toMap("x=x1", "y=y1"))); + assertEquals("e-x1y2",ctest.get("e", toMap("x=x1", "y=y2"))); + } + + public void testVariantInheritanceSimplified() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x","y"}); + QueryProfile x1y2Parent=new QueryProfile("x1y2Parent"); + x1y2Parent.set("c","c-x1y2", (QueryProfileRegistry)null); + test.addInherited(x1y2Parent,new String[] {"x1","y2"}); + test.set("c","c-x1",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + + assertEquals(null,ctest.get("c")); + assertEquals("c-x1",ctest.get("c", toMap("x=x1"))); + assertEquals("c-x1", ctest.get("c", toMap("x=x1", "y=y1"))); + assertEquals("c-x1y2",ctest.get("c", toMap("x=x1", "y=y2"))); + } + + public void testVariantInheritanceWithCompoundReferences() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + test.set("a.b","default-a.b", (QueryProfileRegistry)null); + + QueryProfile ac=new QueryProfile("ac"); + ac.set("a.c","referenced-a.c", (QueryProfileRegistry)null); + test.addInherited(ac,new String[] {"x1"}); + test.set("a.b","x1-a.b",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("Basic functionality","default-a.b",ctest.get("a.b")); + assertEquals("Inherited variance reference works","referenced-a.c",ctest.get("a.c", toMap("x=x1"))); + assertEquals("Inherited variance reference overriding works","x1-a.b",ctest.get("a.b", toMap("x=x1"))); + } + + public void testVariantInheritanceWithTwoLevelCompoundReferencesVariantAtFirstLevel() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + test.set("o.a.b","default-a.b", (QueryProfileRegistry)null); + + QueryProfile ac=new QueryProfile("ac"); + ac.set("o.a.c","referenced-a.c", (QueryProfileRegistry)null); + test.addInherited(ac,new String[] {"x1"}); + test.set("o.a.b","x1-a.b",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("Basic functionality","default-a.b",ctest.get("o.a.b")); + assertEquals("Inherited variance reference works","referenced-a.c",ctest.get("o.a.c", toMap("x=x1"))); + assertEquals("Inherited variance reference overriding works","x1-a.b",ctest.get("o.a.b", toMap("x=x1"))); + } + + public void testVariantInheritanceWithTwoLevelCompoundReferencesVariantAtSecondLevel() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + + QueryProfile ac=new QueryProfile("ac"); + ac.set("a.c","referenced-a.c", (QueryProfileRegistry)null); + test.addInherited(ac,new String[] {"x1"}); + test.set("a.b","x1-a.b",new String[] {"x1"}, null); + + QueryProfile top=new QueryProfile("top"); + top.set("o.a.b","default-a.b", (QueryProfileRegistry)null); + top.set("o",test, (QueryProfileRegistry)null); + + CompiledQueryProfile ctop = top.compile(null); + assertEquals("Basic functionality","default-a.b",ctop.get("o.a.b")); + assertEquals("Inherited variance reference works","referenced-a.c",ctop.get("o.a.c", toMap("x=x1"))); + assertEquals("Inherited variance reference does not override value set in referent","default-a.b",ctop.get("o.a.b", toMap("x=x1"))); // Note: Changed from x1-a.b in 4.2.3 + } + + public void testVariantInheritanceOverridesBaseInheritance1() { + QueryProfile test=new QueryProfile("test"); + QueryProfile baseInherited=new QueryProfile("baseInherited"); + baseInherited.set("a.b","baseInherited-a.b", (QueryProfileRegistry)null); + QueryProfile variantInherited=new QueryProfile("variantInherited"); + variantInherited.set("a.b","variantInherited-a.b", (QueryProfileRegistry)null); + test.setDimensions(new String[] {"x"}); + test.addInherited(baseInherited); + test.addInherited(variantInherited,new String[] {"x1"}); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("baseInherited-a.b",ctest.get("a.b")); + assertEquals("variantInherited-a.b",ctest.get("a.b",toMap("x=x1"))); + } + + public void testVariantInheritanceOverridesBaseInheritance2() { + QueryProfile test=new QueryProfile("test"); + QueryProfile baseInherited=new QueryProfile("baseInherited"); + baseInherited.set("a.b","baseInherited-a.b", (QueryProfileRegistry)null); + QueryProfile variantInherited=new QueryProfile("variantInherited"); + variantInherited.set("a.b","variantInherited-a.b", (QueryProfileRegistry)null); + test.setDimensions(new String[] {"x"}); + test.addInherited(baseInherited); + test.addInherited(variantInherited,new String[] {"x1"}); + test.set("a.c","variant-a.c",new String[] {"x1"}, null); + + CompiledQueryProfile ctest = test.compile(null); + assertEquals("baseInherited-a.b",ctest.get("a.b")); + assertEquals("variantInherited-a.b",ctest.get("a.b", toMap("x=x1"))); + assertEquals("variant-a.c",ctest.get("a.c", toMap("x=x1"))); + } + + public void testVariantInheritanceOverridesBaseInheritanceComplex() { + QueryProfile defaultQP=new QueryProfile("default"); + defaultQP.set("model.defaultIndex","title", (QueryProfileRegistry)null); + + QueryProfile root=new QueryProfile("root"); + root.addInherited(defaultQP); + root.set("model.defaultIndex","default", (QueryProfileRegistry)null); + + QueryProfile querybest=new QueryProfile("querybest"); + querybest.set("defaultIndex","title", (QueryProfileRegistry)null); + querybest.set("queryString","best", (QueryProfileRegistry)null); + + QueryProfile multi=new QueryProfile("multi"); + multi.setDimensions(new String[] {"x"}); + multi.addInherited(defaultQP); + multi.set("model",querybest, (QueryProfileRegistry)null); + multi.addInherited(root,new String[] {"x1"}); + multi.set("model.queryString","love",new String[] {"x1"}, null); + + // Rumtimize + defaultQP.freeze(); + root.freeze(); + querybest.freeze(); + multi.freeze(); + Properties runtime = new QueryProfileProperties(multi.compile(null)); + + assertEquals("default",runtime.get("model.defaultIndex", toMap("x=x1"))); + assertEquals("love",runtime.get("model.queryString", toMap("x=x1"))); + } + + public void testVariantInheritanceOverridesBaseInheritanceComplexSimplified() { + QueryProfile root=new QueryProfile("root"); + root.set("model.defaultIndex","default", (QueryProfileRegistry)null); + + QueryProfile multi=new QueryProfile("multi"); + multi.setDimensions(new String[] {"x"}); + multi.set("model.defaultIndex","title", (QueryProfileRegistry)null); + multi.addInherited(root,new String[] {"x1"}); + + assertEquals("default",multi.compile(null).get("model.defaultIndex", toMap("x=x1"))); + } + + public void testVariantInheritanceOverridesBaseInheritanceMixed() { + QueryProfile root=new QueryProfile("root"); + root.set("model.defaultIndex","default", (QueryProfileRegistry)null); + + QueryProfile multi=new QueryProfile("multi"); + multi.setDimensions(new String[] {"x"}); + multi.set("model.defaultIndex","title", (QueryProfileRegistry)null); + multi.set("model.queryString","modelQuery", (QueryProfileRegistry)null); + multi.addInherited(root,new String[] {"x1"}); + multi.set("model.queryString","modelVariantQuery",new String[] {"x1"}, null); + + CompiledQueryProfile cmulti = multi.compile(null); + assertEquals("default",cmulti.get("model.defaultIndex", toMap("x=x1"))); + assertEquals("modelVariantQuery",cmulti.get("model.queryString", toMap("x=x1"))); + } + + public void testListVariantPropertiesNoCompounds() { + QueryProfile parent1=new QueryProfile("parent1"); + parent1.set("a","parent1-a", (QueryProfileRegistry)null); // Defined everywhere + parent1.set("b","parent1-b", (QueryProfileRegistry)null); // Defined everywhere, but no variants + parent1.set("c","parent1-c", (QueryProfileRegistry)null); // Defined in both parents only + + QueryProfile parent2=new QueryProfile("parent2"); + parent2.set("a","parent2-a", (QueryProfileRegistry)null); + parent2.set("b","parent2-b", (QueryProfileRegistry)null); + parent2.set("c","parent2-c", (QueryProfileRegistry)null); + parent2.set("d","parent2-d", (QueryProfileRegistry)null); // Defined in second parent only + + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x","y"}); + main.addInherited(parent1); + main.addInherited(parent2); + main.set("a","main-a", (QueryProfileRegistry)null); + main.set("a","main-a-x1",new String[] {"x1"}, null); + main.set("e","main-e-x1",new String[] {"x1"}, null); // Defined in two variants only + main.set("f","main-f-x1",new String[] {"x1"}, null); // Defined in one variants only + main.set("a","main-a-x1.y1",new String[] {"x1","y1"}, null); + main.set("a","main-a-x1.y2",new String[] {"x1","y2"}, null); + main.set("e","main-e-x1.y2",new String[] {"x1","y2"}, null); + main.set("g","main-g-x1.y2",new String[] {"x1","y2"}, null); // Defined in one variant only + main.set("b","main-b", (QueryProfileRegistry)null); + + QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); + inheritedVariant1.set("a","inheritedVariant1-a", (QueryProfileRegistry)null); + inheritedVariant1.set("h","inheritedVariant1-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + + QueryProfile inheritedVariant2=new QueryProfile("inheritedVariant2"); + inheritedVariant2.set("a","inheritedVariant2-a", (QueryProfileRegistry)null); + inheritedVariant2.set("h","inheritedVariant2-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + inheritedVariant2.set("i","inheritedVariant2-i", (QueryProfileRegistry)null); // Only defined in one inherited variant + + QueryProfile inheritedVariant3=new QueryProfile("inheritedVariant3"); + inheritedVariant3.set("j","inheritedVariant3-j", (QueryProfileRegistry)null); // Only defined in one inherited variant, but inherited twice + + main.addInherited(inheritedVariant1,new String[] {"x1"}); + main.addInherited(inheritedVariant3,new String[] {"x1"}); + main.addInherited(inheritedVariant2,new String[] {"x1","y2"}); + main.addInherited(inheritedVariant3,new String[] {"x1","y2"}); + + // Runtime-ify + Properties properties=new QueryProfileProperties(main.compile(null)); + + int expectedBaseSize=4; + + // No context + Map<String,Object> listed=properties.listProperties(); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + + // Context x=x1 + listed=properties.listProperties(toMap(main, new String[] {"x1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + assertEquals("main-e-x1",listed.get("e")); + assertEquals("main-f-x1",listed.get("f")); + assertEquals("inheritedVariant1-h",listed.get("h")); + assertEquals("inheritedVariant3-j",listed.get("j")); + + // Context x=x1,y=y1 + listed=properties.listProperties(toMap(main, new String[] {"x1","y1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1.y1",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + assertEquals("main-e-x1",listed.get("e")); + assertEquals("main-f-x1",listed.get("f")); + assertEquals("inheritedVariant1-h",listed.get("h")); + assertEquals("inheritedVariant3-j",listed.get("j")); + + // Context x=x1,y=y2 + listed=properties.listProperties(toMap(main, new String[] {"x1","y2"})); + assertEquals(expectedBaseSize+6,listed.size()); + assertEquals("main-a-x1.y2",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + assertEquals("main-e-x1.y2",listed.get("e")); + assertEquals("main-f-x1",listed.get("f")); + assertEquals("main-g-x1.y2",listed.get("g")); + assertEquals("inheritedVariant2-h",listed.get("h")); + assertEquals("inheritedVariant2-i",listed.get("i")); + assertEquals("inheritedVariant3-j",listed.get("j")); + + // Context x=x1,y=y3 + listed=properties.listProperties(toMap(main, new String[] {"x1","y3"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + assertEquals("main-e-x1",listed.get("e")); + assertEquals("main-f-x1",listed.get("f")); + assertEquals("inheritedVariant1-h",listed.get("h")); + assertEquals("inheritedVariant3-j",listed.get("j")); + + // Context x=x2,y=y1 + listed=properties.listProperties(toMap(main, new String[] {"x2","y1"})); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("a")); + assertEquals("main-b",listed.get("b")); + assertEquals("parent1-c",listed.get("c")); + assertEquals("parent2-d",listed.get("d")); + } + + public void testListVariantPropertiesCompounds1Simplified() { + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x","y"}); + main.set("a.p1","main-a-x1",new String[] {"x1"}, null); + + QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); + inheritedVariant1.set("a.p1","inheritedVariant1-a", (QueryProfileRegistry)null); + main.addInherited(inheritedVariant1,new String[] {"x1"}); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + // Context x=x1 + Map<String,Object> listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals("main-a-x1",listed.get("a.p1")); + } + + public void testListVariantPropertiesCompounds1() { + QueryProfile parent1=new QueryProfile("parent1"); + parent1.set("a.p1","parent1-a", (QueryProfileRegistry)null); // Defined everywhere + parent1.set("b.p1","parent1-b", (QueryProfileRegistry)null); // Defined everywhere, but no variants + parent1.set("c.p1","parent1-c", (QueryProfileRegistry)null); // Defined in both parents only + + QueryProfile parent2=new QueryProfile("parent2"); + parent2.set("a.p1","parent2-a", (QueryProfileRegistry)null); + parent2.set("b.p1","parent2-b", (QueryProfileRegistry)null); + parent2.set("c.p1","parent2-c", (QueryProfileRegistry)null); + parent2.set("d.p1","parent2-d", (QueryProfileRegistry)null); // Defined in second parent only + + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x","y"}); + main.addInherited(parent1); + main.addInherited(parent2); + main.set("a.p1","main-a", (QueryProfileRegistry)null); + main.set("a.p1","main-a-x1",new String[] {"x1"}, null); + main.set("e.p1","main-e-x1",new String[] {"x1"}, null); // Defined in two variants only + main.set("f.p1","main-f-x1",new String[] {"x1"}, null); // Defined in one variants only + main.set("a.p1","main-a-x1.y1",new String[] {"x1","y1"}, null); + main.set("a.p1","main-a-x1.y2",new String[] {"x1","y2"}, null); + main.set("e.p1","main-e-x1.y2",new String[] {"x1","y2"}, null); + main.set("g.p1","main-g-x1.y2",new String[] {"x1","y2"}, null); // Defined in one variant only + main.set("b.p1","main-b", (QueryProfileRegistry)null); + + QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); + inheritedVariant1.set("a.p1","inheritedVariant1-a", (QueryProfileRegistry)null); + inheritedVariant1.set("h.p1","inheritedVariant1-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + + QueryProfile inheritedVariant2=new QueryProfile("inheritedVariant2"); + inheritedVariant2.set("a.p1","inheritedVariant2-a", (QueryProfileRegistry)null); + inheritedVariant2.set("h.p1","inheritedVariant2-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + inheritedVariant2.set("i.p1","inheritedVariant2-i", (QueryProfileRegistry)null); // Only defined in one inherited variant + + QueryProfile inheritedVariant3=new QueryProfile("inheritedVariant3"); + inheritedVariant3.set("j.p1","inheritedVariant3-j", (QueryProfileRegistry)null); // Only defined in one inherited variant, but inherited twice + + main.addInherited(inheritedVariant1,new String[] {"x1"}); + main.addInherited(inheritedVariant3,new String[] {"x1"}); + main.addInherited(inheritedVariant2,new String[] {"x1","y2"}); + main.addInherited(inheritedVariant3,new String[] {"x1","y2"}); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + int expectedBaseSize=4; + + // No context + Map<String,Object> listed=properties.listProperties(); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + + // Context x=x1 + listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + assertEquals("main-e-x1",listed.get("e.p1")); + assertEquals("main-f-x1",listed.get("f.p1")); + assertEquals("inheritedVariant1-h",listed.get("h.p1")); + assertEquals("inheritedVariant3-j",listed.get("j.p1")); + + // Context x=x1,y=y1 + listed=properties.listProperties(toMap(main,new String[] {"x1","y1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1.y1",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + assertEquals("main-e-x1",listed.get("e.p1")); + assertEquals("main-f-x1",listed.get("f.p1")); + assertEquals("inheritedVariant1-h",listed.get("h.p1")); + assertEquals("inheritedVariant3-j",listed.get("j.p1")); + + // Context x=x1,y=y2 + listed=properties.listProperties(toMap(main,new String[] {"x1","y2"})); + assertEquals(expectedBaseSize+6,listed.size()); + assertEquals("main-a-x1.y2",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + assertEquals("main-e-x1.y2",listed.get("e.p1")); + assertEquals("main-f-x1",listed.get("f.p1")); + assertEquals("main-g-x1.y2",listed.get("g.p1")); + assertEquals("inheritedVariant2-h",listed.get("h.p1")); + assertEquals("inheritedVariant2-i",listed.get("i.p1")); + assertEquals("inheritedVariant3-j",listed.get("j.p1")); + + // Context x=x1,y=y3 + listed=properties.listProperties(toMap(main,new String[] {"x1","y3"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + assertEquals("main-e-x1",listed.get("e.p1")); + assertEquals("main-f-x1",listed.get("f.p1")); + assertEquals("inheritedVariant1-h",listed.get("h.p1")); + assertEquals("inheritedVariant3-j",listed.get("j.p1")); + + // Context x=x2,y=y1 + listed=properties.listProperties(toMap(main,new String[] {"x2","y1"})); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("a.p1")); + assertEquals("main-b",listed.get("b.p1")); + assertEquals("parent1-c",listed.get("c.p1")); + assertEquals("parent2-d",listed.get("d.p1")); + } + + public void testListVariantPropertiesCompounds2() { + QueryProfile parent1=new QueryProfile("parent1"); + parent1.set("p1.a","parent1-a", (QueryProfileRegistry)null); // Defined everywhere + parent1.set("p1.b","parent1-b", (QueryProfileRegistry)null); // Defined everywhere, but no variants + parent1.set("p1.c","parent1-c", (QueryProfileRegistry)null); // Defined in both parents only + + QueryProfile parent2=new QueryProfile("parent2"); + parent2.set("p1.a","parent2-a", (QueryProfileRegistry)null); + parent2.set("p1.b","parent2-b", (QueryProfileRegistry)null); + parent2.set("p1.c","parent2-c", (QueryProfileRegistry)null); + parent2.set("p1.d","parent2-d", (QueryProfileRegistry)null); // Defined in second parent only + + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x","y"}); + main.addInherited(parent1); + main.addInherited(parent2); + main.set("p1.a","main-a", (QueryProfileRegistry)null); + main.set("p1.a","main-a-x1",new String[] {"x1"}, null); + main.set("p1.e","main-e-x1",new String[] {"x1"}, null); // Defined in two variants only + main.set("p1.f","main-f-x1",new String[] {"x1"}, null); // Defined in one variants only + main.set("p1.a","main-a-x1.y1",new String[] {"x1","y1"}, null); + main.set("p1.a","main-a-x1.y2",new String[] {"x1","y2"}, null); + main.set("p1.e","main-e-x1.y2",new String[] {"x1","y2"}, null); + main.set("p1.g","main-g-x1.y2",new String[] {"x1","y2"}, null); // Defined in one variant only + main.set("p1.b","main-b", (QueryProfileRegistry)null); + + QueryProfile inheritedVariant1=new QueryProfile("inheritedVariant1"); + inheritedVariant1.set("p1.a","inheritedVariant1-a", (QueryProfileRegistry)null); + inheritedVariant1.set("p1.h","inheritedVariant1-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + + QueryProfile inheritedVariant2=new QueryProfile("inheritedVariant2"); + inheritedVariant2.set("p1.a","inheritedVariant2-a", (QueryProfileRegistry)null); + inheritedVariant2.set("p1.h","inheritedVariant2-h", (QueryProfileRegistry)null); // Only defined in two inherited variants + inheritedVariant2.set("p1.i","inheritedVariant2-i", (QueryProfileRegistry)null); // Only defined in one inherited variant + + QueryProfile inheritedVariant3=new QueryProfile("inheritedVariant3"); + inheritedVariant3.set("p1.j","inheritedVariant3-j", (QueryProfileRegistry)null); // Only defined in one inherited variant, but inherited twice + + main.addInherited(inheritedVariant1,new String[] {"x1"}); + main.addInherited(inheritedVariant3,new String[] {"x1"}); + main.addInherited(inheritedVariant2,new String[] {"x1","y2"}); + main.addInherited(inheritedVariant3,new String[] {"x1","y2"}); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + int expectedBaseSize=4; + + // No context + Map<String,Object> listed=properties.listProperties(); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + + // Context x=x1 + listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + assertEquals("main-e-x1",listed.get("p1.e")); + assertEquals("main-f-x1",listed.get("p1.f")); + assertEquals("inheritedVariant1-h",listed.get("p1.h")); + assertEquals("inheritedVariant3-j",listed.get("p1.j")); + + // Context x=x1,y=y1 + listed=properties.listProperties(toMap(main,new String[] {"x1","y1"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1.y1",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + assertEquals("main-e-x1",listed.get("p1.e")); + assertEquals("main-f-x1",listed.get("p1.f")); + assertEquals("inheritedVariant1-h",listed.get("p1.h")); + assertEquals("inheritedVariant3-j",listed.get("p1.j")); + + // Context x=x1,y=y2 + listed=properties.listProperties(toMap(main,new String[] {"x1","y2"})); + assertEquals(expectedBaseSize+6,listed.size()); + assertEquals("main-a-x1.y2",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + assertEquals("main-e-x1.y2",listed.get("p1.e")); + assertEquals("main-f-x1",listed.get("p1.f")); + assertEquals("main-g-x1.y2",listed.get("p1.g")); + assertEquals("inheritedVariant2-h",listed.get("p1.h")); + assertEquals("inheritedVariant2-i",listed.get("p1.i")); + assertEquals("inheritedVariant3-j",listed.get("p1.j")); + + // Context x=x1,y=y3 + listed=properties.listProperties(toMap(main,new String[] {"x1","y3"})); + assertEquals(expectedBaseSize+4,listed.size()); + assertEquals("main-a-x1",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + assertEquals("main-e-x1",listed.get("p1.e")); + assertEquals("main-f-x1",listed.get("p1.f")); + assertEquals("inheritedVariant1-h",listed.get("p1.h")); + assertEquals("inheritedVariant3-j",listed.get("p1.j")); + + // Context x=x2,y=y1 + listed=properties.listProperties(toMap(main,new String[] {"x2","y1"})); + assertEquals(expectedBaseSize,listed.size()); + assertEquals("main-a",listed.get("p1.a")); + assertEquals("main-b",listed.get("p1.b")); + assertEquals("parent1-c",listed.get("p1.c")); + assertEquals("parent2-d",listed.get("p1.d")); + } + + public void testQueryProfileReferences() { + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x1"}); + QueryProfile referencedMain=new QueryProfile("referencedMain"); + referencedMain.set("r1","mainReferenced-r1", (QueryProfileRegistry)null); // In both + referencedMain.set("r2","mainReferenced-r2", (QueryProfileRegistry)null); // Only in this + QueryProfile referencedVariant=new QueryProfile("referencedVariant"); + referencedVariant.set("r1","variantReferenced-r1", (QueryProfileRegistry)null); // In both + referencedVariant.set("r3","variantReferenced-r3", (QueryProfileRegistry)null); // Only in this + + main.set("a",referencedMain, (QueryProfileRegistry)null); + main.set("a",referencedVariant,new String[] {"x1"}, null); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + // No context + Map<String,Object> listed=properties.listProperties(); + assertEquals(2,listed.size()); + assertEquals("mainReferenced-r1",listed.get("a.r1")); + assertEquals("mainReferenced-r2",listed.get("a.r2")); + + // Context x=x1 + listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals(3,listed.size()); + assertEquals("variantReferenced-r1",listed.get("a.r1")); + assertEquals("mainReferenced-r2",listed.get("a.r2")); + assertEquals("variantReferenced-r3",listed.get("a.r3")); + } + + public void testQueryProfileReferencesWithSubstitution() { + QueryProfile main=new QueryProfile("main"); + main.setDimensions(new String[] {"x1"}); + QueryProfile referencedMain=new QueryProfile("referencedMain"); + referencedMain.set("r1","%{prefix}mainReferenced-r1", (QueryProfileRegistry)null); // In both + referencedMain.set("r2","%{prefix}mainReferenced-r2", (QueryProfileRegistry)null); // Only in this + QueryProfile referencedVariant=new QueryProfile("referencedVariant"); + referencedVariant.set("r1","%{prefix}variantReferenced-r1", (QueryProfileRegistry)null); // In both + referencedVariant.set("r3","%{prefix}variantReferenced-r3", (QueryProfileRegistry)null); // Only in this + + main.set("a",referencedMain, (QueryProfileRegistry)null); + main.set("a",referencedVariant,new String[] {"x1"}, null); + main.set("prefix","mainPrefix:", (QueryProfileRegistry)null); + main.set("prefix","variantPrefix:",new String[] {"x1"}, null); + + Properties properties=new QueryProfileProperties(main.compile(null)); + + // No context + Map<String,Object> listed=properties.listProperties(); + assertEquals(3,listed.size()); + assertEquals("mainPrefix:mainReferenced-r1",listed.get("a.r1")); + assertEquals("mainPrefix:mainReferenced-r2",listed.get("a.r2")); + + // Context x=x1 + listed=properties.listProperties(toMap(main,new String[] {"x1"})); + assertEquals(4,listed.size()); + assertEquals("variantPrefix:variantReferenced-r1",listed.get("a.r1")); + assertEquals("variantPrefix:mainReferenced-r2",listed.get("a.r2")); + assertEquals("variantPrefix:variantReferenced-r3",listed.get("a.r3")); + } + + public void testNewsCase1() { + QueryProfile shortcuts=new QueryProfile("shortcuts"); + shortcuts.setDimensions(new String[] {"custid_1","custid_2","custid_3","custid_4","custid_5","custid_6"}); + shortcuts.set("testout","outside", (QueryProfileRegistry)null); + shortcuts.set("test.out","dotoutside", (QueryProfileRegistry)null); + shortcuts.set("testin","inside",new String[] {"yahoo","ca","sc"}, null); + shortcuts.set("test.in","dotinside",new String[] {"yahoo","ca","sc"}, null); + + QueryProfile profile=new QueryProfile("default"); + profile.setDimensions(new String[] {"custid_1","custid_2","custid_3","custid_4","custid_5","custid_6"}); + profile.addInherited(shortcuts, new String[] {"yahoo",null,"sc"}); + + profile.freeze(); + Query query = new Query(HttpRequest.createTestRequest("?query=test&custid_1=yahoo&custid_2=ca&custid_3=sc", Method.GET), profile.compile(null)); + + assertEquals("outside",query.properties().get("testout")); + assertEquals("dotoutside",query.properties().get("test.out")); + assertEquals("inside",query.properties().get("testin")); + assertEquals("dotinside",query.properties().get("test.in")); + } + + public void testNewsCase2() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions("sort,resulttypes,rss,age,intl,testid".split(",")); + String[] dimensionValues=new String[] {null,null,"0"}; + test.set("discovery","sources",dimensionValues, null); + test.set("discoverytypes","article",dimensionValues, null); + test.set("discovery.sources.count","10",dimensionValues, null); + + CompiledQueryProfile ctest = test.compile(null); + + assertEquals("sources",ctest.get("discovery", toMap(test, dimensionValues))); + assertEquals("article",ctest.get("discoverytypes", toMap(test, dimensionValues))); + assertEquals("10",ctest.get("discovery.sources.count", toMap(test, dimensionValues))); + + Map<String,Object> values=ctest.listValues("",toMap(test,dimensionValues)); + assertEquals(3,values.size()); + assertEquals("sources",values.get("discovery")); + assertEquals("article",values.get("discoverytypes")); + assertEquals("10",values.get("discovery.sources.count")); + + Map<String,Object> sourceValues=ctest.listValues("discovery.sources",toMap(test,dimensionValues)); + assertEquals(1,sourceValues.size()); + assertEquals("10",sourceValues.get("count")); + } + + public void testRuntimeAssignmentInClone() { + QueryProfile test=new QueryProfile("test"); + test.setDimensions(new String[] {"x"}); + String[] x1=new String[] {"x1"}; + Map<String,String> x1m=toMap(test,x1); + test.set("a","30",x1, null); + test.set("a.b","20",x1, null); + test.set("a.b.c","10",x1, null); + + // Setting in one profile works + Query qMain = new Query(HttpRequest.createTestRequest("?query=test", Method.GET), test.compile(null)); + qMain.properties().set("a.b","50",x1m); + assertEquals("50",qMain.properties().get("a.b",x1m)); + + // Cloning + Query qBranch=qMain.clone(); + + // Setting in main still works + qMain.properties().set("a.b","51",x1m); + assertEquals("51",qMain.properties().get("a.b",x1m)); + + // Clone is not affected by change in original + assertEquals("50",qBranch.properties().get("a.b",x1m)); + + // Setting in clone works + qBranch.properties().set("a.b","70",x1m); + assertEquals("70",qBranch.properties().get("a.b",x1m)); + + // Setting in clone does not affect original + assertEquals("51",qMain.properties().get("a.b",x1m)); + } + + public void testIncompatibleDimensions() { + QueryProfile alert = new QueryProfile("alert"); + + QueryProfile backendBase = new QueryProfile("backendBase"); + backendBase.setDimensions(new String[] { "sort", "resulttypes", "rss" }); + backendBase.set("custid", "s", (QueryProfileRegistry)null); + + QueryProfile backend = new QueryProfile("backend"); + backend.setDimensions(new String[] { "sort", "offset", "resulttypes", "rss", "age", "lang", "fr", "entry" }); + backend.addInherited(backendBase); + + QueryProfile web = new QueryProfile("web"); + web.setDimensions(new String[] { "entry", "recency" }); + web.set("fr", "alerts", new String[] { "alert" }, null); + + alert.set("config.backend.vertical.news", backend, (QueryProfileRegistry)null); + alert.set("config.backend.multimedia", web, (QueryProfileRegistry)null); + backend.set("custid", "yahoo/alerts", new String[] { null, null, null, null, null, "en-US", null, "alert"}, null); + + CompiledQueryProfile cAlert = alert.compile(null); + assertEquals("yahoo/alerts", cAlert.get("config.backend.vertical.news.custid", toMap("entry=alert", "intl=us", "lang=en-US"))); + } + + public void testIncompatibleDimensionsSimplified() { + QueryProfile alert = new QueryProfile("alert"); + + QueryProfile backendBase = new QueryProfile("backendBase"); + backendBase.set("custid", "s", (QueryProfileRegistry)null); + + QueryProfile backend = new QueryProfile("backend"); + backend.setDimensions(new String[] { "sort", "lang", "fr", "entry" }); + backend.set("custid", "yahoo/alerts", new String[] { null, "en-US", null, "alert"}, null); + backend.addInherited(backendBase); + + QueryProfile web = new QueryProfile("web"); + web.setDimensions(new String[] { "entry", "recency" }); + web.set("fr", "alerts", new String[] { "alert" }, null); + + alert.set("vertical", backend, (QueryProfileRegistry)null); + alert.set("multimedia", web, (QueryProfileRegistry)null); + + CompiledQueryProfile cAlert = alert.compile(null); + assertEquals("yahoo/alerts", cAlert.get("vertical.custid", toMap("entry=alert", "intl=us", "lang=en-US"))); + } + + private void assertGet(String expectedValue, String parameter, String[] dimensionValues, QueryProfile profile, CompiledQueryProfile cprofile) { + Map<String,String> context=toMap(profile,dimensionValues); + assertEquals("Looking up '" + parameter + "' for '" + Arrays.toString(dimensionValues) + "'",expectedValue,cprofile.get(parameter,context)); + } + + public static Map<String,String> toMap(QueryProfile profile, String[] dimensionValues) { + Map<String,String> context=new HashMap<>(); + List<String> dimensions; + if (profile.getVariants()!=null) + dimensions=profile.getVariants().getDimensions(); + else + dimensions=((BackedOverridableQueryProfile)profile).getBacking().getVariants().getDimensions(); + + for (int i=0; i<dimensionValues.length; i++) + context.put(dimensions.get(i),dimensionValues[i]); // Lookup dim. names to ease test... + return context; + } + + public static final Map<String, String> toMap(String... bindings) { + Map<String, String> context = new HashMap<>(); + for (String binding : bindings) { + String[] entry = binding.split("="); + context.put(entry[0].trim(), entry[1].trim()); + } + return context; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/FieldTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/FieldTypeTestCase.java new file mode 100644 index 00000000000..ae1e39e52d0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/FieldTypeTestCase.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +/** + * @author bratseth + */ +public class FieldTypeTestCase extends junit.framework.TestCase { + + public void testConvertToFromString() { + QueryProfileTypeRegistry registry=new QueryProfileTypeRegistry(); + registry.register(new QueryProfileType("foo")); + + assertEquals("string", FieldType.fromString("string",registry).stringValue()); + assertEquals("boolean", FieldType.fromString("boolean",registry).stringValue()); + assertEquals("query-profile", FieldType.fromString("query-profile",registry).stringValue()); + assertEquals("query-profile:foo", FieldType.fromString("query-profile:foo",registry).stringValue()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java new file mode 100644 index 00000000000..8e2c465911b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/MandatoryTestCase.java @@ -0,0 +1,201 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.search.test.QueryTestCase; + +/** + * @author bratseth + */ +public class MandatoryTestCase extends junit.framework.TestCase { + + private QueryProfileTypeRegistry registry; + + private QueryProfileType type, user; + + protected @Override void setUp() { + type=new QueryProfileType(new ComponentId("testtype")); + user=new QueryProfileType(new ComponentId("user")); + registry=new QueryProfileTypeRegistry(); + registry.register(type); + registry.register(user); + + addTypeFields(type); + addUserFields(user); + } + + private void addTypeFields(QueryProfileType type) { + boolean mandatory=true; + type.addField(new FieldDescription("myString", FieldType.fromString("string",registry), mandatory)); + type.addField(new FieldDescription("myInteger",FieldType.fromString("integer",registry))); + type.addField(new FieldDescription("myLong",FieldType.fromString("long",registry))); + type.addField(new FieldDescription("myFloat",FieldType.fromString("float",registry))); + type.addField(new FieldDescription("myDouble",FieldType.fromString("double",registry))); + type.addField(new FieldDescription("myQueryProfile",FieldType.fromString("query-profile",registry))); + type.addField(new FieldDescription("myUserQueryProfile", FieldType.fromString("query-profile:user",registry),mandatory)); + } + + private void addUserFields(QueryProfileType user) { + boolean mandatory=true; + user.addField(new FieldDescription("myUserString",FieldType.fromString("string",registry),mandatory)); + user.addField(new FieldDescription("myUserInteger",FieldType.fromString("integer",registry),mandatory)); + } + + public void testMandatoryFullySpecifiedQueryProfile() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + test.set("myString","aString", registry); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, registry); + myUser.set("myUserString",1, registry); + test.set("myUserQueryProfile", myUser, registry); + registry.register(myUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Fully specified request + assertError(null, new Query(QueryTestCase.httpEncode("?queryProfile=test"), cRegistry.getComponent("test"))); + } + + public void testMandatoryRequestPropertiesNeeded() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Underspecified request 1 + assertError("Incomplete query: Parameter 'myString' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("", Method.GET), cRegistry.getComponent("test"))); + + // Underspecified request 2 + assertError("Incomplete query: Parameter 'myUserQueryProfile.myUserString' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("?myString=aString", Method.GET), cRegistry.getComponent("test"))); + + // Fully specified request + assertError(null, new Query(HttpRequest.createTestRequest("?myString=aString&myUserQueryProfile.myUserString=userString", Method.GET), cRegistry.getComponent("test"))); + } + + /** Same as above except the whole thing is nested in maps */ + public void testMandatoryNestedInMaps() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile topMap=new QueryProfile("topMap"); + registry.register(topMap); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + registry.register(subMap); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + subMap.set("test",test, registry); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Underspecified request 1 + assertError("Incomplete query: Parameter 'subMap.test.myString' is mandatory in query profile 'topMap' but is not set", + new Query(HttpRequest.createTestRequest("", Method.GET), cRegistry.getComponent("topMap"))); + + // Underspecified request 2 + assertError("Incomplete query: Parameter 'subMap.test.myUserQueryProfile.myUserString' is mandatory in query profile 'topMap' but is not set", + new Query(HttpRequest.createTestRequest("?subMap.test.myString=aString", Method.GET), cRegistry.getComponent("topMap"))); + + // Fully specified request + assertError(null, new Query(HttpRequest.createTestRequest("?subMap.test.myString=aString&subMap.test.myUserQueryProfile.myUserString=userString", Method.GET), cRegistry.getComponent("topMap"))); + } + + /** Here, no user query profile is referenced in the query profile, but one is chosen in the request */ + public void testMandatoryUserProfileSetInRequest() { + QueryProfile test=new QueryProfile("test"); + test.setType(type); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, (QueryProfileRegistry)null); + + QueryProfileRegistry registry = new QueryProfileRegistry(); + registry.register(test); + registry.register(myUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Underspecified request 1 + assertError("Incomplete query: Parameter 'myUserQueryProfile' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("?myString=aString", Method.GET), cRegistry.getComponent("test"))); + + // Underspecified request 1 + assertError("Incomplete query: Parameter 'myUserQueryProfile.myUserString' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("?myString=aString&myUserQueryProfile=user", Method.GET), cRegistry.getComponent("test"))); + + // Fully specified request + assertError(null, new Query(HttpRequest.createTestRequest("?myString=aString&myUserQueryProfile=user&myUserQueryProfile.myUserString=userString", Method.GET), cRegistry.getComponent("test"))); + } + + /** Here, a partially specified query profile is added to a non-mandatory field, making the request underspecified */ + public void testNonMandatoryUnderspecifiedUserProfileSetInRequest() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile test = new QueryProfile("test"); + test.setType(type); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger", 1, registry); + myUser.set("myUserString","userValue", registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + QueryProfile otherUser=new QueryProfile("otherUser"); + otherUser.setType(user); + otherUser.set("myUserInteger", 2, registry); + registry.register(otherUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + // Fully specified request + assertError(null, new Query(HttpRequest.createTestRequest("?myString=aString", Method.GET), cRegistry.getComponent("test"))); + + // Underspecified because an underspecified profile is added + assertError("Incomplete query: Parameter 'myQueryProfile.myUserString' is mandatory in query profile 'test' of type 'testtype' but is not set", + new Query(HttpRequest.createTestRequest("?myString=aString&myQueryProfile=otherUser", Method.GET), cRegistry.getComponent("test"))); + + // Back to fully specified + assertError(null, new Query(HttpRequest.createTestRequest("?myString=aString&myQueryProfile=otherUser&myQueryProfile.myUserString=userString", Method.GET), cRegistry.getComponent("test"))); + } + + private void assertError(String message,Query query) { + assertEquals(message, query.validate()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java new file mode 100644 index 00000000000..562418647c8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.yolean.Exceptions; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; + +/** + * tests creating invalid names + * + * @author bratseth + */ +public class NameTestCase extends junit.framework.TestCase { + + public void testNames() { + assertLegalName("aB"); + assertIllegalName("a."); + assertLegalName("_a_b"); + assertLegalName("a_b"); + assertLegalName("a/b"); + assertLegalName("/a/b"); + assertLegalName("/a/b/"); + assertIllegalName(""); + } + + public void testFieldNames() { + assertLegalFieldName("aB"); + try { + QueryProfile profile=new QueryProfile("test"); + profile.set("a.","anyValue", (QueryProfileRegistry)null); + fail("Should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("'a.' is not a legal compound name. Names can not end with a dot.", e.getMessage()); + } + assertLegalFieldName("_a_b"); + assertLegalFieldName("a_b"); + assertLegalFieldName("a/b"); + assertLegalFieldName("/a/b"); + assertLegalFieldName("/a/b/"); + assertIllegalFieldName(""); + assertIllegalFieldName("aBc.dooEee.ce_d.-some-other.moreHere", + "Could not set 'aBc.dooEee.ce_d.-some-other.moreHere' to 'anyValue'", + "Illegal name '-some-other'"); + } + + private void assertLegalName(String name) { + new QueryProfile(name); + new QueryProfileType(name); + } + + private void assertLegalFieldName(String name) { + new QueryProfile(name).set(name, "value", (QueryProfileRegistry)null); + new FieldDescription(name,FieldType.stringType); + } + + /** Checks that this is illegal both for profiles and types */ + private void assertIllegalName(String name) { + try { + new QueryProfile(name); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + if (!name.isEmpty()) + assertEquals("Illegal name '" + name + "'",e.getMessage()); + } + + try { + new QueryProfileType(name); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + if (!name.isEmpty()) + assertEquals("Illegal name '" + name + "'",e.getMessage()); + } + } + + private void assertIllegalFieldName(String name) { + assertIllegalFieldName(name,"Could not set '" + name + "' to 'anyValue'","Illegal name '" + name + "'"); + } + + /** Checks that this is illegal both for profiles and types */ + private void assertIllegalFieldName(String name, String expectedHighError, String expectedLowError) { + try { + QueryProfile profile=new QueryProfile("test"); + profile.set(name, "anyValue", (QueryProfileRegistry)null); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals(expectedHighError + ": " + expectedLowError, Exceptions.toMessageString(e)); + } + + try { + new FieldDescription(name, FieldType.stringType); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals(expectedLowError, e.getMessage()); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java new file mode 100644 index 00000000000..77e733a740a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NativePropertiesTestCase.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.yolean.Exceptions; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertThat; + +/** + * Tests that properties can not be set even if they are native, if declared not settable in the query profile + * + * @author bratseth + */ +public class NativePropertiesTestCase extends junit.framework.TestCase { + + public void testNativeInStrict() { + QueryProfileType strictType=new QueryProfileType("strict"); + strictType.setStrict(true); + QueryProfile strict=new QueryProfile("profile"); + strict.setType(strictType); + + try { + new Query(HttpRequest.createTestRequest("?hits=10&tracelevel=5", Method.GET), strict.compile(null)); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + } + + try { + new Query(HttpRequest.createTestRequest("?notnative=5", Method.GET), strict.compile(null)); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString( + "Could not set 'notnative' to '5':" + + " 'notnative' is not declared in query profile type 'strict', and the type is strict")); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/OverrideTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/OverrideTestCase.java new file mode 100644 index 00000000000..77c3d26f9be --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/OverrideTestCase.java @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.component.ComponentId; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +/** + * Tests overriding of field values + * + * @author bratseth + */ +public class OverrideTestCase extends junit.framework.TestCase { + + private QueryProfileTypeRegistry registry; + + private QueryProfileType type, user; + + protected @Override void setUp() { + type=new QueryProfileType(new ComponentId("testtype")); + user=new QueryProfileType(new ComponentId("user")); + registry=new QueryProfileTypeRegistry(); + registry.register(type); + registry.register(user); + + addTypeFields(type); + addUserFields(user); + } + + private void addTypeFields(QueryProfileType type) { + boolean overridable=true; + type.addField(new FieldDescription("myString", FieldType.fromString("string",registry),false,!overridable)); + type.addField(new FieldDescription("myInteger",FieldType.fromString("integer",registry))); + type.addField(new FieldDescription("myLong",FieldType.fromString("long",registry))); + type.addField(new FieldDescription("myFloat",FieldType.fromString("float",registry))); + type.addField(new FieldDescription("myDouble",FieldType.fromString("double",registry))); + type.addField(new FieldDescription("myQueryProfile",FieldType.fromString("query-profile",registry))); + type.addField(new FieldDescription("myUserQueryProfile", FieldType.fromString("query-profile:user",registry),false,!overridable)); + } + + private void addUserFields(QueryProfileType user) { + boolean overridable=true; + user.addField(new FieldDescription("myUserString",FieldType.fromString("string",registry),false,!overridable)); + user.addField(new FieldDescription("myUserInteger",FieldType.fromString("integer",registry))); + } + + /** Check that a simple non-overridable string cannot be overridden */ + public void testSimpleUnoverridable() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile test=new QueryProfile("test"); + test.setType(type); + test.set("myString","finalString", (QueryProfileRegistry)null); + registry.register(test); + registry.freeze(); + + // Assert request assignment does not work + Query query = new Query(HttpRequest.createTestRequest("?myString=newValue", Method.GET), registry.compile().getComponent("test")); + assertEquals(0,query.errors().size()); + assertEquals("finalString",query.properties().get("myString")); + + // Assert direct assignment does not work + query.properties().set("myString","newValue"); + assertEquals("finalString",query.properties().get("myString")); + } + + /** Check that a query profile cannot be overridden */ + public void testUnoverridableQueryProfile() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile test = new QueryProfile("test"); + test.setType(type); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserInteger",1, registry); + myUser.set("myUserString","userValue", registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + QueryProfile otherUser = new QueryProfile("otherUser"); + otherUser.setType(user); + otherUser.set("myUserInteger", 2, registry); + registry.register(otherUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?myUserQueryprofile=otherUser", Method.GET), cRegistry.getComponent("test")); + assertEquals(0,query.errors().size()); + assertEquals(1,query.properties().get("myUserQueryProfile.myUserInteger")); + } + + /** Check that non-overridables are protected also in nested untyped references */ + public void testUntypedNestedUnoverridable() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile topMap = new QueryProfile("topMap"); + registry.register(topMap); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + registry.register(subMap); + + QueryProfile test = new QueryProfile("test"); + test.setType(type); + subMap.set("test",test, registry); + registry.register(test); + + QueryProfile myUser=new QueryProfile("user"); + myUser.setType(user); + myUser.set("myUserString","finalValue", registry); + test.set("myUserQueryProfile",myUser, registry); + registry.register(myUser); + + registry.freeze(); + Query query = new Query(HttpRequest.createTestRequest("?subMap.test.myUserQueryProfile.myUserString=newValue", Method.GET), registry.compile().getComponent("topMap")); + assertEquals(0,query.errors().size()); + assertEquals("finalValue",query.properties().get("subMap.test.myUserQueryProfile.myUserString")); + + query.properties().set("subMap.test.myUserQueryProfile.myUserString","newValue"); + assertEquals("finalValue",query.properties().get("subMap.test.myUserQueryProfile.myUserString")); + } + + /** Tests overridability in an inherited field */ + public void testInheritedNonOverridableInType() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + test.set("myString","finalString", (QueryProfileRegistry)null); + registry.register(test); + + QueryProfile profile=new QueryProfile("profile"); + profile.addInherited(test); + registry.register(profile); + + registry.freeze(); + + Query query = new Query(HttpRequest.createTestRequest("?myString=newString", Method.GET), registry.compile().getComponent("test")); + + assertEquals(0,query.errors().size()); + assertEquals("finalString",query.properties().get("myString")); + + query.properties().set("myString","newString"); + assertEquals("finalString",query.properties().get("myString")); + } + + /** Tests overridability in an inherited field */ + public void testInheritedNonOverridableInProfile() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile test = new QueryProfile("test"); + test.setType(type); + test.set("myInteger", 1, registry); + test.setOverridable("myInteger", false, null); + registry.register(test); + + QueryProfile profile=new QueryProfile("profile"); + profile.addInherited(test); + registry.register(profile); + + registry.freeze(); + + Query query = new Query(HttpRequest.createTestRequest("?myInteger=32", Method.GET), registry.compile().getComponent("test")); + + assertEquals(0,query.errors().size()); + assertEquals(1,query.properties().get("myInteger")); + + query.properties().set("myInteger",32); + assertEquals(1,query.properties().get("myInteger")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/PatchMatchingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/PatchMatchingTestCase.java new file mode 100644 index 00000000000..65a552931ac --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/PatchMatchingTestCase.java @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.types.QueryProfileType; + +/** + * Tests that matching query profiles by path name works + * + * @author bratseth + */ +public class PatchMatchingTestCase extends junit.framework.TestCase { + + public void testPatchMatching() { + QueryProfileType type=new QueryProfileType("type"); + + type.setMatchAsPath(true); + + QueryProfile a=new QueryProfile("a"); + a.setType(type); + QueryProfile abee=new QueryProfile("a/bee"); + abee.setType(type); + abee.addInherited(a); + QueryProfile abeece=new QueryProfile("a/bee/ce"); + abeece.setType(type); + abeece.addInherited(abee); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("a",registry.findQueryProfile("a").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee/eee/").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee/cede").getId().getName()); + assertEquals("a",registry.findQueryProfile("a/foo/bee/cede").getId().getName()); + assertNull(registry.findQueryProfile("abee")); + } + + public void testNoPatchMatching() { + QueryProfileType type=new QueryProfileType("type"); + + type.setMatchAsPath(false); // Default, but set here for clarity + + QueryProfile a=new QueryProfile("a"); + a.setType(type); + QueryProfile abee=new QueryProfile("a/bee"); + abee.setType(type); + abee.addInherited(a); + QueryProfile abeece=new QueryProfile("a/bee/ce"); + abeece.setType(type); + abeece.addInherited(abee); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("a",registry.findQueryProfile("a").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce").getId().getName()); + assertNull(registry.findQueryProfile("a/bee/ce/dee")); // Different from test above + assertNull(registry.findQueryProfile("a/bee/ce/dee/eee/")); // Different from test above + assertNull(registry.findQueryProfile("a/bee/cede")); // Different from test above + assertNull(registry.findQueryProfile("a/foo/bee/cede")); // Different from test above + assertNull(registry.findQueryProfile("abee")); + } + + /** Check that the path matching property is inherited to subtypes */ + public void testPatchMatchingInheritance() { + QueryProfileType type=new QueryProfileType("type"); + QueryProfileType subType=new QueryProfileType("subType"); + subType.inherited().add(type); + + type.setMatchAsPath(true); // Supertype only + + QueryProfile a=new QueryProfile("a"); + a.setType(type); + QueryProfile abee=new QueryProfile("a/bee"); + abee.setType(subType); + abee.addInherited(a); + QueryProfile abeece=new QueryProfile("a/bee/ce"); + abeece.setType(subType); + abeece.addInherited(abee); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("a",registry.findQueryProfile("a").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee/eee/").getId().getName()); + assertEquals("a/bee",registry.findQueryProfile("a/bee/cede").getId().getName()); + assertEquals("a",registry.findQueryProfile("a/foo/bee/cede").getId().getName()); + assertNull(registry.findQueryProfile("abee")); + } + + /** Check that the path matching works with versioned profiles */ + public void testPatchMatchingVersions() { + QueryProfileType type=new QueryProfileType("type"); + + type.setMatchAsPath(true); + + QueryProfile a=new QueryProfile("a"); + a.setType(type); + QueryProfile abee11=new QueryProfile("a/bee:1.1"); + abee11.setType(type); + abee11.addInherited(a); + QueryProfile abee13=new QueryProfile("a/bee:1.3"); + abee13.setType(type); + abee13.addInherited(a); + QueryProfile abeece=new QueryProfile("a/bee/ce"); + abeece.setType(type); + abeece.addInherited(abee13); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee11); + registry.register(abee13); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("a",registry.findQueryProfile("a").getId().getName()); + assertEquals("a/bee:1.1",registry.findQueryProfile("a/bee:1.1").getId().toString()); + assertEquals("a/bee:1.3",registry.findQueryProfile("a/bee").getId().toString()); + assertEquals("a/bee:1.3",registry.findQueryProfile("a/bee:1").getId().toString()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee").getId().getName()); + assertEquals("a/bee/ce",registry.findQueryProfile("a/bee/ce/dee/eee/").getId().getName()); + assertEquals("a/bee:1.1",registry.findQueryProfile("a/bee/cede:1.1").getId().toString()); + assertEquals("a/bee:1.3",registry.findQueryProfile("a/bee/cede").getId().toString()); + assertEquals("a/bee:1.3",registry.findQueryProfile("a/bee/cede:1").getId().toString()); + assertEquals("a",registry.findQueryProfile("a/foo/bee/cede").getId().getName()); + assertNull(registry.findQueryProfile("abee")); + } + + public void testQuirkyNames() { + QueryProfileType type=new QueryProfileType("type"); + + type.setMatchAsPath(true); + + QueryProfile a=new QueryProfile("/a"); + a.setType(type); + QueryProfile abee=new QueryProfile("/a//bee"); + abee.setType(type); + abee.addInherited(a); + QueryProfile abeece=new QueryProfile("/a//bee/ce/"); + abeece.setType(type); + abeece.addInherited(abee); + + QueryProfileRegistry registry=new QueryProfileRegistry(); + registry.register(a); + registry.register(abee); + registry.register(abeece); + registry.freeze(); + + assertNull(registry.findQueryProfile(null)); // No "default" registered + assertEquals("/a",registry.findQueryProfile("/a").getId().getName()); + assertNull(registry.findQueryProfile("a")); + assertEquals("/a//bee",registry.findQueryProfile("/a//bee").getId().getName()); + assertEquals("/a//bee/ce/",registry.findQueryProfile("/a//bee/ce/").getId().getName()); + assertEquals("/a//bee/ce/",registry.findQueryProfile("/a//bee/ce").getId().getName()); + assertEquals("/a//bee/ce/",registry.findQueryProfile("/a//bee/ce/dee").getId().getName()); + assertEquals("/a//bee/ce/",registry.findQueryProfile("/a//bee/ce/dee/eee/").getId().getName()); + assertEquals("/a//bee",registry.findQueryProfile("/a//bee/cede").getId().getName()); + assertEquals("/a",registry.findQueryProfile("/a/foo/bee/cede").getId().getName()); + assertEquals("/a",registry.findQueryProfile("/a/bee").getId().getName()); + assertNull(registry.findQueryProfile("abee")); + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeInheritanceTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeInheritanceTestCase.java new file mode 100644 index 00000000000..85333e1e95a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeInheritanceTestCase.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +/** + * @author bratseth + */ +public class QueryProfileTypeInheritanceTestCase extends junit.framework.TestCase { + + private QueryProfileTypeRegistry registry; + + private QueryProfileType type, typeStrict, user, userStrict; + + protected @Override void setUp() { + type=new QueryProfileType(new ComponentId("testtype")); + typeStrict=new QueryProfileType(new ComponentId("testtypeStrict")); + typeStrict.setStrict(true); + user=new QueryProfileType(new ComponentId("user")); + userStrict=new QueryProfileType(new ComponentId("userStrict")); + userStrict.setStrict(true); + registry=new QueryProfileTypeRegistry(); + registry.register(type); + registry.register(typeStrict); + registry.register(user); + registry.register(userStrict); + + addTypeFields(type); + type.addField(new FieldDescription("myUserQueryProfile", FieldType.fromString("query-profile:user",registry))); + addTypeFields(typeStrict); + typeStrict.addField(new FieldDescription("myUserQueryProfile",FieldType.fromString("query-profile:userStrict",registry))); + addUserFields(user); + addUserFields(userStrict); + } + + private void addTypeFields(QueryProfileType type) { + type.addField(new FieldDescription("myString", FieldType.fromString("string",registry))); + type.addField(new FieldDescription("myInteger",FieldType.fromString("integer",registry))); + type.addField(new FieldDescription("myLong",FieldType.fromString("long",registry))); + type.addField(new FieldDescription("myFloat",FieldType.fromString("float",registry))); + type.addField(new FieldDescription("myDouble",FieldType.fromString("double",registry))); + type.addField(new FieldDescription("myQueryProfile",FieldType.fromString("query-profile",registry))); + } + + private void addUserFields(QueryProfileType user) { + user.addField(new FieldDescription("myUserString",FieldType.fromString("string",registry),true,false)); + user.addField(new FieldDescription("myUserInteger",FieldType.fromString("integer",registry))); + } + + public void testInheritance() { + type.inherited().add(user); + type.freeze(); + user.freeze(); + + assertFalse(type.isOverridable("myUserString")); + assertEquals("myUserInteger", type.getField("myUserInteger").getName()); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + + test.set("myUserInteger","37", (QueryProfileRegistry)null); + test.set("myUnknownInteger","38", (QueryProfileRegistry)null); + CompiledQueryProfile ctest = test.compile(null); + + assertEquals(37, ctest.get("myUserInteger")); + assertEquals("38", ctest.get("myUnknownInteger")); + } + + public void testInheritanceStrict() { + typeStrict.inherited().add(userStrict); + typeStrict.freeze(); + userStrict.freeze(); + + QueryProfile test=new QueryProfile("test"); + test.setType(typeStrict); + + test.set("myUserInteger","37", (QueryProfileRegistry)null); + try { + test.set("myUnknownInteger","38", (QueryProfileRegistry)null); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("'myUnknownInteger' is not declared in query profile type 'testtypeStrict', and the type is strict", + e.getCause().getMessage()); + } + + assertEquals(37,test.get("myUserInteger")); + assertNull(test.get("myUnknownInteger")); + } + + public void testStrictIsInherited() { + type.inherited().add(userStrict); + type.freeze(); + userStrict.freeze(); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + + test.set("myUserInteger","37", (QueryProfileRegistry)null); + try { + test.set("myUnknownInteger","38", (QueryProfileRegistry)null); + fail("Should have failed"); + } + catch (IllegalArgumentException e) { + assertEquals("'myUnknownInteger' is not declared in query profile type 'testtype', and the type is strict", + e.getCause().getMessage()); + } + + CompiledQueryProfile ctest = test.compile(null); + assertEquals(37, ctest.get("myUserInteger")); + assertNull(ctest.get("myUnknownInteger")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java new file mode 100644 index 00000000000..d9dfc733f04 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -0,0 +1,595 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.profile.types.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.tensor.MapTensor; +import com.yahoo.tensor.Tensor; +import com.yahoo.yolean.Exceptions; +import com.yahoo.search.Query; +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; +import com.yahoo.search.query.profile.QueryProfileProperties; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertThat; + +/** + * tests query profiles with/and types + * + * @author bratseth + */ +public class QueryProfileTypeTestCase extends junit.framework.TestCase { + + private QueryProfileRegistry registry; + + private QueryProfileType type, typeStrict, user, userStrict; + + @Override + protected void setUp() { + registry = new QueryProfileRegistry(); + + type = new QueryProfileType(new ComponentId("testtype")); + type.inherited().add(registry.getTypeRegistry().getComponent(new ComponentId("native"))); + typeStrict = new QueryProfileType(new ComponentId("testtypeStrict")); + typeStrict.setStrict(true); + user = new QueryProfileType(new ComponentId("user")); + userStrict = new QueryProfileType(new ComponentId("userStrict")); + userStrict.setStrict(true); + + registry.getTypeRegistry().register(type); + registry.getTypeRegistry().register(typeStrict); + registry.getTypeRegistry().register(user); + registry.getTypeRegistry().register(userStrict); + + addTypeFields(type, registry.getTypeRegistry()); + type.addField(new FieldDescription("myUserQueryProfile",FieldType.fromString("query-profile:user",registry.getTypeRegistry()))); + addTypeFields(typeStrict, registry.getTypeRegistry()); + typeStrict.addField(new FieldDescription("myUserQueryProfile",FieldType.fromString("query-profile:userStrict",registry.getTypeRegistry()))); + addUserFields(user, registry.getTypeRegistry()); + addUserFields(userStrict, registry.getTypeRegistry()); + + } + + private void addTypeFields(QueryProfileType type, QueryProfileTypeRegistry registry) { + type.addField(new FieldDescription("myString", FieldType.fromString("string",registry)), registry); + type.addField(new FieldDescription("myInteger",FieldType.fromString("integer",registry),"int"), registry); + type.addField(new FieldDescription("myLong",FieldType.fromString("long",registry)), registry); + type.addField(new FieldDescription("myFloat",FieldType.fromString("float",registry)), registry); + type.addField(new FieldDescription("myDouble",FieldType.fromString("double",registry)), registry); + type.addField(new FieldDescription("myBoolean",FieldType.fromString("boolean",registry)), registry); + type.addField(new FieldDescription("myBoolean",FieldType.fromString("boolean",registry)), registry); + type.addField(new FieldDescription("ranking.features.query(myTensor1)",FieldType.fromString("tensor",registry)), registry); + type.addField(new FieldDescription("ranking.features.query(myTensor2)",FieldType.fromString("tensor(x[2],y[2])",registry)), registry); + type.addField(new FieldDescription("ranking.features.query(myTensor3)",FieldType.fromString("tensor(x{})",registry)), registry); + type.addField(new FieldDescription("myQuery",FieldType.fromString("query",registry)), registry); + type.addField(new FieldDescription("myQueryProfile",FieldType.fromString("query-profile",registry),"qp"), registry); + } + + private void addUserFields(QueryProfileType user, QueryProfileTypeRegistry registry) { + user.addField(new FieldDescription("myUserString",FieldType.fromString("string",registry)), registry); + user.addField(new FieldDescription("myUserInteger",FieldType.fromString("integer",registry),"uint"), registry); + } + + public void testTypedOfPrimitivesAssignmentNonStrict() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + registry.register(profile); + + profile.set("myString","anyValue", registry); + profile.set("nontypedString", "anyValueToo", registry); // legal because this is not strict + assertWrongType(profile,"integer","myInteger","notInteger"); + assertWrongType(profile, "integer", "myInteger", "1.5"); + profile.set("myInteger", 3, registry); + assertWrongType(profile,"long","myLong","notLong"); + assertWrongType(profile, "long", "myLong", "1.5"); + profile.set("myLong", 4000000000000l, registry); + assertWrongType(profile, "float", "myFloat", "notFloat"); + profile.set("myFloat", 3.14f, registry); + assertWrongType(profile, "double", "myDouble", "notDouble"); + profile.set("myDouble",2.18, registry); + profile.set("myBoolean",true, registry); + + String tensorString1 = "{{a:a1, b:b1}:1.0, {a:a2}:2.0}}"; + profile.set("ranking.features.query(myTensor1)", tensorString1, registry); + String tensorString2 = "{{x:0, y:0}:1.0, {x:0, y:1}:2.0}}"; + profile.set("ranking.features.query(myTensor2)", tensorString2, registry); + String tensorString3 = "{{x:x1}:1.0, {x:x2}:2.0}}"; + profile.set("ranking.features.query(myTensor3)", tensorString3, registry); + + profile.set("myQuery", "...", registry); // TODO + profile.set("myQueryProfile.anyString","value1", registry); + profile.set("myQueryProfile.anyDouble",8.76, registry); + profile.set("myUserQueryProfile.myUserString","value2", registry); + profile.set("myUserQueryProfile.anyString", "value3", registry); // Legal because user is not strict + assertWrongType(profile, "integer", "myUserQueryProfile.myUserInteger", "notInteger"); + profile.set("myUserQueryProfile.uint",1337, registry); // Set using alias + profile.set("myUserQueryProfile.anyDouble", 9.13, registry); // Legal because user is not strict + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + QueryProfileProperties properties = new QueryProfileProperties(cRegistry.findQueryProfile("test")); + + assertEquals("anyValue", properties.get("myString")); + assertEquals("anyValueToo", properties.get("nontypedString")); + assertEquals(3, properties.get("myInteger")); + assertEquals(3, properties.get("Int")); + assertEquals(4000000000000l, properties.get("myLong")); + assertEquals(3.14f, properties.get("myFloat")); + assertEquals(2.18, properties.get("myDouble")); + assertEquals(true, properties.get("myBoolean")); + assertEquals(Tensor.from(tensorString1), properties.get("ranking.features.query(myTensor1)")); + assertEquals(Tensor.from("tensor(x[2],y[2])", tensorString2), properties.get("ranking.features.query(myTensor2)")); + assertEquals(Tensor.from("tensor(x{})", tensorString3), properties.get("ranking.features.query(myTensor3)")); + // TODO: assertEquals(..., cprofile.get("myQuery")); + assertEquals("value1", properties.get("myQueryProfile.anyString")); + assertEquals("value1", properties.get("QP.anyString")); + assertEquals(8.76, properties.get("myQueryProfile.anyDouble")); + assertEquals(8.76, properties.get("qp.anyDouble")); + assertEquals("value2", properties.get("myUserQueryProfile.myUserString")); + assertEquals("value3", properties.get("myUserQueryProfile.anyString")); + assertEquals(1337, properties.get("myUserQueryProfile.myUserInteger")); + assertEquals(1337, properties.get("myUserQueryProfile.uint")); + assertEquals(9.13, properties.get("myUserQueryProfile.anyDouble")); + assertNull(properties.get("nonExisting")); + + properties.set("INt", 51); + assertEquals(51, properties.get("InT")); + assertEquals(51, properties.get("myInteger")); + } + + public void testTypedOfPrimitivesAssignmentStrict() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(typeStrict); + + profile.set("myString", "anyValue", registry); + assertNotPermitted(profile, "nontypedString", "anyValueToo"); // Illegal because this is strict + assertWrongType(profile,"integer","myInteger","notInteger"); + assertWrongType(profile, "integer", "myInteger", "1.5"); + profile.set("myInteger", 3, registry); + assertWrongType(profile,"long","myLong","notLong"); + assertWrongType(profile, "long", "myLong", "1.5"); + profile.set("myLong", 4000000000000l, registry); + assertWrongType(profile, "float", "myFloat", "notFloat"); + profile.set("myFloat", 3.14f, registry); + assertWrongType(profile, "double", "myDouble", "notDouble"); + profile.set("myDouble",2.18, registry); + profile.set("myQueryProfile.anyString","value1", registry); + profile.set("myQueryProfile.anyDouble",8.76, registry); + profile.set("myUserQueryProfile.myUserString", "value2", registry); + assertNotPermitted(profile, "myUserQueryProfile.anyString", "value3"); // Illegal because this is strict + assertWrongType(profile, "integer", "myUserQueryProfile.myUserInteger", "notInteger"); + profile.set("myUserQueryProfile.myUserInteger", 1337, registry); + assertNotPermitted(profile, "myUserQueryProfile.anyDouble", 9.13); // Illegal because this is strict + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("anyValue", cprofile.get("myString")); + assertNull(cprofile.get("nontypedString")); + assertEquals(3, cprofile.get("myInteger")); + assertEquals(4000000000000l, cprofile.get("myLong")); + assertEquals(3.14f, cprofile.get("myFloat")); + assertEquals(2.18, cprofile.get("myDouble")); + assertEquals("value1", cprofile.get("myQueryProfile.anyString")); + assertEquals(8.76, cprofile.get("myQueryProfile.anyDouble")); + assertEquals("value2", cprofile.get("myUserQueryProfile.myUserString")); + assertNull(cprofile.get("myUserQueryProfile.anyString")); + assertEquals(1337, cprofile.get("myUserQueryProfile.myUserInteger")); + assertNull(cprofile.get("myUserQueryProfile.anyDouble")); + } + + /** Tests assigning a subprofile directly */ + public void testTypedAssignmentOfQueryProfilesNonStrict() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + + QueryProfile map1=new QueryProfile("myMap1"); + map1.set("key1","value1", registry); + + QueryProfile map2=new QueryProfile("myMap2"); + map2.set("key2","value2", registry); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + + assertWrongType(profile,"reference to a query profile","myQueryProfile","aString"); + profile.set("myQueryProfile",map1, registry); + profile.set("someMap",map2, registry); // Legal because this is not strict + assertWrongType(profile,"reference to a query profile of type 'user'","myUserQueryProfile",map1); + profile.set("myUserQueryProfile",myUser, registry); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("value1", cprofile.get("myQueryProfile.key1")); + assertEquals("value2", cprofile.get("someMap.key2")); + assertEquals("userValue1", cprofile.get("myUserQueryProfile.myUserString")); + assertEquals(442, cprofile.get("myUserQueryProfile.myUserInteger")); + } + + /** Tests assigning a subprofile directly */ + public void testTypedAssignmentOfQueryProfilesStrict() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(typeStrict); + + QueryProfile map1=new QueryProfile("myMap1"); + map1.set("key1","value1", registry); + + QueryProfile map2=new QueryProfile("myMap2"); + map2.set("key2","value2", registry); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(userStrict); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + + assertWrongType(profile,"reference to a query profile","myQueryProfile","aString"); + profile.set("myQueryProfile",map1, registry); + assertNotPermitted(profile,"someMap",map2); + assertWrongType(profile,"reference to a query profile of type 'userStrict'","myUserQueryProfile",map1); + profile.set("myUserQueryProfile",myUser, registry); + + CompiledQueryProfile cprofile = profile.compile(null); + + assertEquals("value1", cprofile.get("myQueryProfile.key1")); + assertNull(cprofile.get("someMap.key2")); + assertEquals("userValue1", cprofile.get("myUserQueryProfile.myUserString")); + assertEquals(442, cprofile.get("myUserQueryProfile.myUserInteger")); + } + + /** Tests assigning a subprofile as an id string */ + public void testTypedAssignmentOfQueryProfileReferencesNonStrict() { + QueryProfile profile = new QueryProfile("test"); + profile.setType(type); + + QueryProfile map1 = new QueryProfile("myMap1"); + map1.set("key1","value1", registry); + + QueryProfile map2 = new QueryProfile("myMap2"); + map2.set("key2","value2", registry); + + QueryProfile myUser = new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + + registry.register(profile); + registry.register(map1); + registry.register(map2); + registry.register(myUser); + + assertWrongType(profile,"reference to a query profile", "myQueryProfile", "aString"); + registry.register(map1); + profile.set("myQueryProfile", "myMap1", registry); + registry.register(map2); + profile.set("someMap", "myMap2", registry); // NOTICE: Will set as a string because we cannot know this is a reference + assertWrongType(profile, "reference to a query profile of type 'user'", "myUserQueryProfile", "myMap1"); + registry.register(myUser); + profile.set("myUserQueryProfile","myUser", registry); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cprofile = cRegistry.getComponent("test"); + + assertEquals("value1", cprofile.get("myQueryProfile.key1")); + assertEquals("myMap2", cprofile.get("someMap")); + assertNull("Asking for an value which cannot be completely resolved returns null", cprofile.get("someMap.key2")); + assertEquals("userValue1", cprofile.get("myUserQueryProfile.myUserString")); + assertEquals(442, cprofile.get("myUserQueryProfile.myUserInteger")); + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here there exists a user profile already, and then a new one is overwritten + */ + public void testTypedOverridingOfQueryProfileReferencesNonStrictThroughQuery() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + QueryProfileRegistry registry = new QueryProfileRegistry(); + registry.register(profile); + registry.register(myUser); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cprofile = cRegistry.getComponent("test"); + + Query query = new Query(HttpRequest.createTestRequest("?myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cprofile); + + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1", query.properties().get("myUserQueryProfile.myUserString")); + assertEquals(845, query.properties().get("myUserQueryProfile.myUserInteger")); + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here no user profile is set before it is assigned in the query + */ + public void testTypedAssignmentOfQueryProfileReferencesNonStrictThroughQuery() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(profile); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + CompiledQueryProfile cprofile = cRegistry.getComponent("test"); + + Query query = new Query(HttpRequest.createTestRequest("?myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cprofile); + + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1", query.properties().get("myUserQueryProfile.myUserString")); + assertEquals(845, query.properties().get("myUserQueryProfile.myUserInteger")); + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here no user profile is set before it is assigned in the query + */ + public void testTypedAssignmentOfQueryProfileReferencesStrictThroughQuery() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(typeStrict); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(userStrict); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(profile); + registry.register(newUser); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1",query.properties().get("myUserQueryProfile.myUserString")); + assertEquals(845,query.properties().get("myUserQueryProfile.myUserInteger")); + + try { + query.properties().set("myUserQueryProfile.someKey","value"); + fail("Should not be allowed to set this"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'myUserQueryProfile.someKey' to 'value': 'someKey' is not declared in query profile type 'userStrict', and the type is strict", + Exceptions.toMessageString(e)); + } + + } + + public void testTensorRankFeatureInRequest() throws UnsupportedEncodingException { + QueryProfile profile=new QueryProfile("test"); + profile.setType(type); + registry.register(profile); + + CompiledQueryProfileRegistry cRegistry = registry.compile(); + String tensorString = "{{a:a1, b:b1}:1.0, {a:a2}:2.0}}"; + Query query = new Query(HttpRequest.createTestRequest("?" + encode("ranking.features.query(myTensor1)") + + "=" + encode(tensorString), + com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("test")); + assertEquals(0, query.errors().size()); + assertEquals(MapTensor.from(tensorString), query.properties().get("ranking.features.query(myTensor1)")); + assertEquals(MapTensor.from(tensorString), query.getRanking().getFeatures().getTensor("query(myTensor1)").get()); + } + + private String encode(String s) throws UnsupportedEncodingException { + return URLEncoder.encode(s, "utf8"); + } + + public void testIllegalStrictAssignmentFromRequest() { + QueryProfile profile=new QueryProfile("test"); + profile.setType(typeStrict); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(userStrict); + + profile.set("myUserQueryProfile", newUser, registry); + + try { + new Query( + HttpRequest.createTestRequest( + "?myUserQueryProfile.nondeclared=someValue", + com.yahoo.jdisc.http.HttpRequest.Method.GET), + profile.compile(null)); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString("Could not set 'myUserQueryProfile.nondeclared' to 'someValue': 'nondeclared' is not declared in query profile type 'userStrict', and the type is strict")); + } + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here there exists a user profile already, and then a new one is overwritten. + * The whole thing is accessed through a two levels of nontyped top-level profiles + */ + public void testTypedOverridingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { + QueryProfile topMap=new QueryProfile("topMap"); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + subMap.set("typeProfile",test, registry); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + test.set("myUserQueryProfile",myUser, registry); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(topMap); + registry.register(subMap); + registry.register(test); + registry.register(myUser); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); + + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1", query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserString")); + assertEquals(845, query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserInteger")); + } + + /** + * Same as previous test but using the untyped myQueryProfile reference instead of the typed myUserQueryProfile + */ + public void testAnonTypedOverridingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { + QueryProfile topMap=new QueryProfile("topMap"); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + subMap.set("typeProfile",test, registry); + + QueryProfile myUser=new QueryProfile("myUser"); + myUser.setType(user); + myUser.set("myUserString","userValue1", registry); + myUser.set("myUserInteger",442, registry); + test.set("myQueryProfile",myUser, registry); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(topMap); + registry.register(subMap); + registry.register(test); + registry.register(myUser); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1",query.properties().get("subMap.typeProfile.myQueryProfile.myUserString")); + assertEquals(845,query.properties().get("subMap.typeProfile.myQueryProfile.myUserInteger")); + } + + /** + * Tests setting a illegal value in a strict profile nested under untyped maps + */ + public void testSettingValueInStrictTypeNestedUnderUntypedMaps() { + QueryProfile topMap=new QueryProfile("topMap"); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + + QueryProfile test=new QueryProfile("test"); + test.setType(typeStrict); + subMap.set("typeProfile",test, registry); + + registry.register(topMap); + registry.register(subMap); + registry.register(test); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + try { + new Query( + HttpRequest.createTestRequest( + "?subMap.typeProfile.someValue=value", + com.yahoo.jdisc.http.HttpRequest.Method.GET), + cRegistry.getComponent("topMap")); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString("Could not set 'subMap.typeProfile.someValue' to 'value': 'someValue' is not declared in query profile type 'testtypeStrict', and the type is strict")); + } + } + + /** + * Tests overriding a subprofile as an id string through the query. + * Here, no user profile is set before it is assigned in the query + * The whole thing is accessed through a two levels of nontyped top-level profiles + */ + public void testTypedSettingOfQueryProfileReferencesNonStrictThroughQueryNestedInAnUntypedProfile() { + QueryProfile topMap=new QueryProfile("topMap"); + + QueryProfile subMap=new QueryProfile("topSubMap"); + topMap.set("subMap",subMap, registry); + + QueryProfile test=new QueryProfile("test"); + test.setType(type); + subMap.set("typeProfile",test, registry); + + QueryProfile newUser=new QueryProfile("newUser"); + newUser.setType(user); + newUser.set("myUserString","newUserValue1", registry); + newUser.set("myUserInteger",845, registry); + + registry.register(topMap); + registry.register(subMap); + registry.register(test); + registry.register(newUser); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + + Query query = new Query(HttpRequest.createTestRequest("?subMap.typeProfile.myUserQueryProfile=newUser", com.yahoo.jdisc.http.HttpRequest.Method.GET), cRegistry.getComponent("topMap")); + assertEquals(0, query.errors().size()); + + assertEquals("newUserValue1", query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserString")); + assertEquals(845, query.properties().get("subMap.typeProfile.myUserQueryProfile.myUserInteger")); + } + + private void assertWrongType(QueryProfile profile,String typeName,String name,Object value) { + try { + profile.set(name,value, registry); + fail("Should fail setting " + name + " to " + value); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set '" + name + "' to '" + value + "': '" + value + "' is not a " + typeName, + Exceptions.toMessageString(e)); + } + } + + private void assertNotPermitted(QueryProfile profile,String name,Object value) { + String localName = new CompoundName(name).last(); + try { + profile.set(name, value, registry); + fail("Should fail setting " + name + " to " + value); + } + catch (IllegalArgumentException e) { + assertTrue(Exceptions.toMessageString(e).startsWith("Could not set '" + name + "' to '" + value + "': '" + localName + "' is not declared")); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java new file mode 100644 index 00000000000..d68745b0d57 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/properties/test/PropertyMapTestCase.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.properties.test; + +import com.yahoo.processing.request.properties.PropertyMap; + +import java.util.Collections; +import java.util.List; + +/** + * @author bratseth + */ +public class PropertyMapTestCase extends junit.framework.TestCase { + + public void testCloning() { + PropertyMap map=new PropertyMap(); + map.set("clonable",new ClonableObject()); + map.set("nonclonable",new NonClonableObject()); + map.set("clonableArray",new ClonableObject[] {new ClonableObject()}); + map.set("nonclonableArray",new NonClonableObject[] {new NonClonableObject()}); + map.set("clonableList", Collections.singletonList(new ClonableObject())); + map.set("nonclonableList", Collections.singletonList(new NonClonableObject())); + assertNotNull(map.get("clonable")); + assertNotNull(map.get("nonclonable")); + + PropertyMap mapClone=map.clone(); + assertTrue(map.get("clonable") != mapClone.get("clonable")); + assertTrue(map.get("nonclonable") == mapClone.get("nonclonable")); + + assertTrue(map.get("clonableArray") != mapClone.get("clonableArray")); + assertTrue(first(map.get("clonableArray")) != first(mapClone.get("clonableArray"))); + assertTrue(first(map.get("nonclonableArray")) == first(mapClone.get("nonclonableArray"))); + } + + private Object first(Object object) { + if (object instanceof Object[]) + return ((Object[])object)[0]; + if (object instanceof List) + return ((List<?>)object).get(0); + throw new IllegalArgumentException(); + } + + public static class ClonableObject implements Cloneable { + + @Override + public ClonableObject clone() { + try { + return (ClonableObject)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException(e); + } + } + + } + + private static class NonClonableObject { + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/test/RequestContextPropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/test/RequestContextPropertiesTestCase.java new file mode 100644 index 00000000000..ff924bb59ea --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/properties/test/RequestContextPropertiesTestCase.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.properties.test; + +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.test.QueryTestCase; + +/** + * Tests that dimension arguments in queries are transferred correctly to dimension values + * + * @author bratseth + */ +public class RequestContextPropertiesTestCase extends junit.framework.TestCase { + + public void testIt() { + QueryProfile p=new QueryProfile("test"); + p.setDimensions(new String[] {"x"}); + p.set("a","a-default", (QueryProfileRegistry)null); + p.set("a","a-x1",new String[] {"x1"}, null); + p.set("a","a-+x1",new String[] {"+x1"}, null); + Query q1 = new Query(QueryTestCase.httpEncode("?query=foo"), p.compile(null)); + assertEquals("a-default",q1.properties().get("a")); + Query q2 = new Query(QueryTestCase.httpEncode("?query=foo&x=x1"),p.compile(null)); + assertEquals("a-x1",q2.properties().get("a")); + Query q3 = new Query(QueryTestCase.httpEncode("?query=foo&x=+x1"),p.compile(null)); + assertEquals("a-+x1",q3.properties().get("a")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java new file mode 100644 index 00000000000..35185b8d8f6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.properties.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.Arrays; +import java.util.HashSet; + +import com.yahoo.processing.request.properties.PropertyMap; +import org.junit.Test; + +import com.yahoo.search.query.properties.SubProperties; + +/** + * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + */ +public class SubPropertiesTestCase { + + @Test + public void testSubProperties() { + PropertyMap map = new PropertyMap() {{ + set("a.e","1"); + set("a.f",2); + set("b.e","3"); + set("f",3); + set("e","2"); + set("d","a"); + }}; + + SubProperties sub = new SubProperties("a", map); + assertEquals("1",sub.get("e")); + assertEquals(2,sub.get("f")); + assertNull(sub.get("d")); + assertEquals(new HashSet<>(Arrays.asList("e", "f")), sub.listProperties("").keySet()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java new file mode 100644 index 00000000000..8f3ac661d0f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/RewriterFeaturesTestCase.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.parser.SpecialTokenRegistry; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.Execution.Context; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens; + +/** + * Fine grained testing of RewriterFeatures for easier testing of innards. + */ +public class RewriterFeaturesTestCase { + + private static final String ASCII_ELLIPSIS = "..."; + + @Test + public final void testConvertStringToQTree() { + Execution placeholder = new Execution(Context.createContextStub()); + SpecialTokenRegistry tokenRegistry = new SpecialTokenRegistry( + new SpecialtokensConfig( + new SpecialtokensConfig.Builder() + .tokenlist(new Tokenlist.Builder().name( + "default").tokens( + new Tokens.Builder().token(ASCII_ELLIPSIS))))); + placeholder.context().setTokenRegistry(tokenRegistry); + Query query = new Query(); + query.getModel().setExecution(placeholder); + Item parsed = RewriterFeatures.convertStringToQTree(query, "a b c " + + ASCII_ELLIPSIS); + assertSame(AndItem.class, parsed.getClass()); + assertEquals(4, ((CompositeItem) parsed).getItemCount()); + assertEquals(ASCII_ELLIPSIS, ((CompositeItem) parsed).getItem(3).toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/GenericExpansionRewriterTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/GenericExpansionRewriterTestCase.java new file mode 100644 index 00000000000..c89ca16a265 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/GenericExpansionRewriterTestCase.java @@ -0,0 +1,202 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import java.util.*; +import java.io.File; + +import com.yahoo.search.searchchain.*; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; +import com.yahoo.search.query.rewrite.RewritesConfig; + +/** + * Test Cases for GenericExpansionRewriter + * + * @author karenlee@yahoo-inc.com + */ +public class GenericExpansionRewriterTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + private final String CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_generic_expansion_rewriter.cfg"; + private final String GENERIC_EXPAND_DICT_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "generic_expansion.fsa"; + private final String REWRITER_NAME = GenericExpansionRewriter.REWRITER_NAME; + + /** + * Load the GenericExpansionRewriterSearcher and prepare the + * execution object + */ + protected void setUp() { + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(CONFIG_PATH); + HashMap<String, File> fileList = new HashMap<>(); + fileList.put(GenericExpansionRewriter.GENERIC_EXPAND_DICT, new File(GENERIC_EXPAND_DICT_PATH)); + GenericExpansionRewriter searcher = new GenericExpansionRewriter(config, fileList); + + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searcher); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public GenericExpansionRewriterTestCase(String name) { + super(name); + } + + /** + * MaxRewrites=3, PartialPhraseMatch is on, type=adv case + */ + public void testPartialPhraseMaxRewriteAdvType() { + utils.assertRewrittenQuery("?query=(modern new york city travel phone number) OR (travel agency) OR travel&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true&" + + REWRITER_NAME + "." + RewriterConstants.MAX_REWRITES + "=3", + "query 'OR (AND modern (OR (AND rewrite11 rewrite12) rewrite2 rewrite3 " + + "(AND new york city travel)) (OR pn (AND phone number))) (OR ta (AND travel agency)) " + + "(OR tr travel)'"); + } + + /** + * PartialPhraseMatch is off, type=adv case + */ + public void testPartialPhraseNoMaxRewriteAdvType() { + utils.assertRewrittenQuery("?query=(modern new york city travel phone number) OR (travel agency) OR travel&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'OR (AND modern new york city travel phone number) " + + "(OR ta (AND travel agency)) (OR tr travel)'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, type=adv, added filter case + */ + public void testFullPhraseNoMaxRewriteAdvTypeFilter() { + utils.assertRewrittenQuery("?query=ca OR (modern new york city travel phone number) OR (travel agency) OR travel&" + + "type=adv&filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'RANK (OR (OR california ca) (AND modern new york city travel phone number) " + + "(OR ta (AND travel agency)) (OR tr travel)) |citystate:santa |clara |ca'"); + } + + /** + * MaxRewrites=0 (i.e No MaxRewrites), PartialPhraseMatch is on, type=adv, added filter case + */ + public void testPartialPhraseNoMaxRewriteAdvTypeFilter() { + utils.assertRewrittenQuery("?query=ca OR (modern new york city travel phone number) OR (travel agency) OR travel&" + + "type=adv&filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.MAX_REWRITES + "=0", + "query 'RANK (OR (OR california ca) (AND modern (OR \"rewrite11 rewrite12\" " + + "rewrite2 rewrite3 rewrite4 rewrite5 (AND new york city travel)) " + + "(OR pn (AND phone number))) (OR ta (AND travel agency)) (OR tr travel)) " + + "|citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, single word, added filter case + */ + public void testFullPhraseNoMaxRewriteSingleWordFilter() { + utils.assertRewrittenQuery("?query=ca&" + + "filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'RANK (OR california ca) |citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is on, single word, added filter case + */ + public void testPartialPhraseNoMaxRewriteSingleWordFilter() { + utils.assertRewrittenQuery("?query=ca&" + + "filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'RANK (OR california ca) |citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, multi word, added filter case + */ + public void testFullPhraseNoMaxRewriteMultiWordFilter() { + utils.assertRewrittenQuery("?query=travel agency&" + + "filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'RANK (OR ta (AND travel agency)) |citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is on, multi word, added filter case + */ + public void testPartialPhraseNoMaxRewriteMultiWordFilter() { + utils.assertRewrittenQuery("?query=modern new york city travel phone number&" + + "filter=citystate:santa clara ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'RANK (AND modern (OR (AND rewrite11 rewrite12) rewrite2 rewrite3 " + + "rewrite4 rewrite5 (AND new york city travel)) (OR pn (AND phone number))) " + + "|citystate:santa |clara |ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, single word + */ + public void testFullPhraseNoMaxRewriteSingleWord() { + utils.assertRewrittenQuery("?query=ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'OR california ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is on, single word + */ + public void testPartialPhraseNoMaxRewriteSingleWord() { + utils.assertRewrittenQuery("?query=ca&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'OR california ca'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, multi word + */ + public void testFullPhraseNoMaxRewriteMultiWord() { + utils.assertRewrittenQuery("?query=travel agency&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'OR ta (AND travel agency)'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is off, multi word, no full match + */ + public void testFullPhraseNoMaxRewriteMultiWordNoMatch() { + utils.assertRewrittenQuery("?query=nyc travel agency&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=false", + "query 'AND nyc travel agency'"); + } + + /** + * No MaxRewrites, PartialPhraseMatch is on, multi word + */ + public void testPartialPhraseNoMaxRewriteMultiWord() { + utils.assertRewrittenQuery("?query=modern new york city travel phone number&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'AND modern (OR (AND rewrite11 rewrite12) rewrite2 rewrite3 rewrite4 rewrite5 "+ + "(AND new york city travel)) (OR pn (AND phone number))'"); + } + + /** + * Matching multiple word in RANK subtree + * Dictionary contain the word "travel agency", the word "agency" and the word "travel" + * Should rewrite travel but not travel agency in this case + */ + public void testPartialPhraseMultiWordRankTree() { + utils.assertRewrittenQuery("?query=travel RANK agency&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'RANK (OR tr travel) agency'"); + } + + /** + * Matching multiple word in RANK subtree + * Dictionary contain the word "travel agency", the word "agency" and the word "travel" + * Should rewrite travel but not travel agency in this case + */ + public void testFullPhraseMultiWordRankTree() { + utils.assertRewrittenQuery("?query=travel RANK agency&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.PARTIAL_PHRASE_MATCH + "=true", + "query 'RANK (OR tr travel) agency'"); + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/MisspellRewriterTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/MisspellRewriterTestCase.java new file mode 100644 index 00000000000..b5b4acff459 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/MisspellRewriterTestCase.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import com.yahoo.search.*; +import com.yahoo.search.searchchain.*; +import com.yahoo.search.intent.model.*; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; + +/** + * Test Cases for MisspellRewriter + * + * @author karenlee@yahoo-inc.com + */ +public class MisspellRewriterTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + public final String REWRITER_NAME = MisspellRewriter.REWRITER_NAME; + + /** + * Load the QueryRewriteSearcher and prepare the + * execution object + */ + protected void setUp() { + MisspellRewriter searcher = new MisspellRewriter(); + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searcher); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public MisspellRewriterTestCase(String name) { + super(name); + } + + /** + * QSSRewrite and QSSSuggest are on + * QLAS returns spell correction: qss_rw=0.9 qss_sugg=1.0 + */ + public void testQSSRewriteQSSSuggestWithRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("will smith rw", 0.9, + true, false), + utils.createInterpretation("will smith sugg", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&" + + REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true", + "query 'OR (AND willl smith) (AND will smith sugg)'", + intentModel); + } + + /** + * QSSRewrite is on + * QLAS returns spell correction: qss_rw=0.9 qss_rw=0.9 qss_sugg=1.0 + */ + public void testQSSRewriteWithRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("will smith rw1", 0.9, + true, false), + utils.createInterpretation("will smith rw2", 0.9, + true, false), + utils.createInterpretation("will smith sugg", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&" + + REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true", + "query 'OR (AND willl smith) (AND will smith rw1)'", + intentModel); + } + + /** + * QSSSuggest is on + * QLAS returns spell correction: qss_rw=1.0 qss_sugg=0.9 qss_sugg=0.8 + */ + public void testQSSSuggWithRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("will smith rw", 1.0, + true, false), + utils.createInterpretation("will smith sugg1", 0.9, + false, true), + utils.createInterpretation("will smith sugg2", 0.8, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&" + + REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true", + "query 'OR (AND willl smith) (AND will smith sugg1)'", + intentModel); + } + + /** + * QSSRewrite and QSSSuggest are off + * QLAS returns spell correction: qss_rw=1.0 qss_sugg=1.0 + */ + public void testFeautureOffWithRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("will smith rw", 1.0, + true, false), + utils.createInterpretation("will smith sugg", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith", + "query 'AND willl smith'", + intentModel); + } + + /** + * QSSRewrite and QSSSuggest are on + * QLAS returns no spell correction + */ + public void testQSSRewriteQSSSuggWithoutRewrite() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("use diff query for testing", 1.0, + false, false), + utils.createInterpretation("use diff query for testing", 1.0, + false, false)); + + utils.assertRewrittenQuery("?query=will+smith&" + + REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true", + "query 'AND will smith'", + intentModel); + } + + /** + * IntentModel is null + * It should throw exception + */ + public void testNullIntentModelException() { + try { + RewriterUtils.getSpellCorrected(new Query("willl smith"), true, true); + fail(); + } catch (RuntimeException e) { + } + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/NameRewriterTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/NameRewriterTestCase.java new file mode 100644 index 00000000000..ecd798caacd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/NameRewriterTestCase.java @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import java.util.*; +import java.io.File; + +import com.yahoo.search.searchchain.*; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; +import com.yahoo.search.query.rewrite.RewritesConfig; + +/** + * Test Cases for NameRewriter + * + * @author karenlee@yahoo-inc.com + */ +public class NameRewriterTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + private final String CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_name_rewriter.cfg"; + private final String NAME_ENTITY_EXPAND_DICT_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "name_rewriter_entity.fsa"; + private final String REWRITER_NAME = NameRewriter.REWRITER_NAME; + + /** + * Load the NameRewriterSearcher and prepare the + * execution object + */ + protected void setUp() { + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(CONFIG_PATH); + HashMap<String, File> fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(NAME_ENTITY_EXPAND_DICT_PATH)); + NameRewriter searcher = new NameRewriter(config, fileList); + + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searcher); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public NameRewriterTestCase(String name) { + super(name); + } + + /** + * RewritesAsEquiv and OriginalAsUnit are on + */ + public void testRewritesAsEquivAndOriginalAsUnit() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT + "=true", + "query 'OR \"will smith\" (AND will smith movies) " + + "(AND will smith news) (AND will smith imdb) " + + "(AND will smith lyrics) (AND will smith dead) " + + "(AND will smith nfl) (AND will smith new movie hancock) " + + "(AND will smith biography)'"); + } + + /** + * RewritesAsEquiv is on + */ + public void testRewritesAsEquiv() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&", + "query 'OR (AND will smith) (AND will smith movies) " + + "(AND will smith news) (AND will smith imdb) " + + "(AND will smith lyrics) (AND will smith dead) " + + "(AND will smith nfl) (AND will smith new movie hancock) " + + "(AND will smith biography)'"); + } + + /** + * Complex query with more than two levels for RewritesAsEquiv is on case + * Should not rewrite + */ + public void testComplextQueryRewritesAsEquiv() { + utils.assertRewrittenQuery("?query=((will smith) OR (willl smith)) AND (tom cruise)&type=adv&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&", + "query 'AND (OR (AND will smith) (AND willl smith)) (AND tom cruise)'"); + } + + /** + * Single word query for RewritesAsEquiv and OriginalAsUnit on case + */ + public void testSingleWordForRewritesAsEquivAndOriginalAsUnit() { + utils.assertRewrittenQuery("?query=obama&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT + "=true", + "query 'OR obama (AND obama \"nobel peace prize\") " + + "(AND obama wiki) (AND obama nobel prize) " + + "(AND obama nobel peace prize) (AND obama wears mom jeans) " + + "(AND obama sucks) (AND obama news) (AND malia obama) " + + "(AND obama speech) (AND obama nobel) (AND obama wikipedia) " + + "(AND barack obama biography) (AND obama snl) " + + "(AND obama peace prize) (AND michelle obama) (AND barack obama)'"); + } + + /** + * RewritesAsUnitEquiv and OriginalAsUnitEquiv are on + */ + public void testRewritesAsUnitEquivAndOriginalAsUnitEquiv() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + + "=true", + "query 'OR (AND will smith) \"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'"); + } + + /** + * Single word query for RewritesAsUnitEquiv and OriginalAsUnitEquiv on case + */ + public void testSingleWordForRewritesAsUnitEquivAndOriginalAsUnitEquiv() { + utils.assertRewrittenQuery("?query=obama&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + + "=true", + "query 'OR obama \"obama nobel peace prize\" " + + "\"obama wiki\" \"obama nobel prize\" " + + "\"obama wears mom jeans\" " + + "\"obama sucks\" \"obama news\" \"malia obama\" " + + "\"obama speech\" \"obama nobel\" \"obama wikipedia\" " + + "\"barack obama biography\" \"obama snl\" " + + "\"obama peace prize\" \"michelle obama\" \"barack obama\"'"); + } + + /** + * Boosting only query (n/a as rewrite in FSA) + * for RewritesAsEquiv and OriginalAsUnit on case + */ + public void testBoostingQueryForRewritesAsEquivAndOriginalAsUnit() { + utils.assertRewrittenQuery("?query=angelina jolie&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT + "=true", + "query '\"angelina jolie\"'"); + } + + /** + * No match in FSA for the query + * RewritesAsEquiv and OriginalAsUnit on case + */ + public void testFSANoMatchForRewritesAsEquivAndOriginalAsUnit() { + utils.assertRewrittenQuery("?query=tom cruise&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_EQUIV + "=true&" + + REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT + "=true", + "query 'AND tom cruise'"); + } + + /** + * RewritesAsUnitEquiv is on + */ + public void testRewritesAsUnitEquiv() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true", + "query 'OR (AND will smith) \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'"); + } + + /** + * RewritesAsUnitEquiv is on and MaxRewrites is set to 2 + */ + public void testRewritesAsUnitEquivAndMaxRewrites() { + utils.assertRewrittenQuery("?query=will smith&" + + REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + REWRITER_NAME + "." + RewriterConstants.MAX_REWRITES + "=2", + "query 'OR (AND will smith) \"will smith movies\" " + + "\"will smith news\"'"); + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestCase.java new file mode 100644 index 00000000000..f3eaf6ae582 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestCase.java @@ -0,0 +1,133 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import java.io.File; +import java.util.*; + +import com.yahoo.search.*; +import com.yahoo.search.searchchain.*; +import com.yahoo.search.intent.model.*; +import com.yahoo.search.query.rewrite.RewritesConfig; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; + +/** + * Generic Test Cases for QueryRewriteSearcher + * + * @author karenlee@yahoo-inc.com + */ +public class QueryRewriteSearcherTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + private final String NAME_REWRITER_CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_name_rewriter.cfg"; + private final String FAKE_FSA_CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_rewriter_fake_fsa.cfg"; + private final String NAME_ENTITY_EXPAND_DICT_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "name_rewriter_entity.fsa"; + private final String FAKE_FSA_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_name_rewriter.cfg"; + private final String NAME_REWRITER_NAME = NameRewriter.REWRITER_NAME; + private final String MISSPELL_REWRITER_NAME = MisspellRewriter.REWRITER_NAME; + + /** + * Load the QueryRewriteSearcher and prepare the + * execution object + */ + protected void setUp() { + // Instantiate Name Rewriter + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(NAME_REWRITER_CONFIG_PATH); + HashMap<String, File> fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(NAME_ENTITY_EXPAND_DICT_PATH)); + NameRewriter nameRewriter = new NameRewriter(config, fileList); + + // Instantiate Misspell Rewriter + MisspellRewriter misspellRewriter = new MisspellRewriter(); + + // Create a chain of two rewriters + ArrayList<Searcher> searchers = new ArrayList<>(); + searchers.add(misspellRewriter); + searchers.add(nameRewriter); + + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searchers); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public QueryRewriteSearcherTestCase(String name) { + super(name); + } + + /** + * Invalid FSA config path + * Query will be passed to next rewriter + */ + public void testInvalidFSAConfigPath() { + // Instantiate Name Rewriter with fake FSA path + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(FAKE_FSA_CONFIG_PATH); + HashMap<String, File> fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(FAKE_FSA_PATH)); + NameRewriter nameRewriterWithFakePath = new NameRewriter(config, fileList); + + // Instantiate Misspell Rewriter + MisspellRewriter misspellRewriter = new MisspellRewriter(); + + // Create a chain of two rewriters + ArrayList<Searcher> searchers = new ArrayList<>(); + searchers.add(misspellRewriter); + searchers.add(nameRewriterWithFakePath); + + Execution execution = QueryRewriteSearcherTestUtils.createExecutionObj(searchers); + QueryRewriteSearcherTestUtils utilsWithFakePath = new QueryRewriteSearcherTestUtils(execution); + + utilsWithFakePath.assertRewrittenQuery("?query=will smith&" + + NAME_REWRITER_NAME + "." + + RewriterConstants.REWRITES_AS_UNIT_EQUIV + "=true", + "query 'AND will smith'"); + } + + /** + * IntentModel is null and rewriter throws exception + * It should skip to the next rewriter + */ + public void testExceptionInRewriter() { + utils.assertRewrittenQuery("?query=will smith&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + "=true", + "query 'OR (AND will smith) " + + "\"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'"); + } + + /** + * Two rewrites in chain + * Query will be rewritten twice + */ + public void testTwoRewritersInChain() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("wills smith", 0.9, + true, false), + utils.createInterpretation("will smith", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + "=true", + "query 'OR (AND willl smith) (AND will smith) " + + "\"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'", + intentModel); + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java new file mode 100644 index 00000000000..74322a4d980 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/QueryRewriteSearcherTestUtils.java @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import com.yahoo.search.test.QueryTestCase; +import junit.framework.Assert; +import java.util.*; + +import com.yahoo.search.*; +import com.yahoo.search.searchchain.*; +import com.yahoo.search.query.rewrite.RewritesConfig; +import com.yahoo.search.intent.model.*; +import com.yahoo.text.interpretation.Modification; +import com.yahoo.text.interpretation.Interpretation; +import com.yahoo.text.interpretation.Annotations; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.component.chain.Chain; + +/** + * Test utilities for QueryRewriteSearcher + * + * @author karenlee@yahoo-inc.com + */ +public class QueryRewriteSearcherTestUtils { + + private Execution execution; + + /** + * Constructor for this class + * Load the QueryRewriteSearcher and prepare the + * execution object + */ + public QueryRewriteSearcherTestUtils(Execution execution) { + this.execution = execution; + } + + + /** + * Create config object based on config path + * + * @param configPath path for the searcher config + */ + public static RewritesConfig createConfigObj(String configPath) { + ConfigGetter<RewritesConfig> getter = new ConfigGetter<>(RewritesConfig.class); + RewritesConfig config = getter.getConfig(configPath); + return config; + } + + /** + * Create execution object based on searcher + * + * @param searcher searcher to be added to the search chain + */ + public static Execution createExecutionObj(Searcher searcher) { + @SuppressWarnings("deprecation") + Chain<Searcher> searchChain = new Chain<>(searcher); + Execution myExecution = new Execution(searchChain, Execution.Context.createContextStub()); + return myExecution; + } + + /** + * Create execution object based on searchers + * + * @param searchers list of searchers to be added to the search chain + */ + public static Execution createExecutionObj(List<Searcher> searchers) { + @SuppressWarnings("deprecation") + Chain<Searcher> searchChain = new Chain<>(searchers); + Execution myExecution = new Execution(searchChain, Execution.Context.createContextStub()); + return myExecution; + } + + /** + * Compare the rewritten query returned after executing + * the origQuery against the provided finalQuery + * @param origQuery query to be passed to Query object + * e.g. "?query=will%20smith" + * @param finalQuery expected final query from result.getQuery() + * e.g. "query 'AND will smith'" + */ + public void assertRewrittenQuery(String origQuery, String finalQuery) { + Query query = new Query(QueryTestCase.httpEncode(origQuery)); + Result result = execution.search(query); + Assert.assertEquals(finalQuery, result.getQuery().toString()); + } + + /** + * Set the provided intent model + * Compare the rewritten query returned after executing + * the origQuery against the provided finalQuery + * @param origQuery query to be passed to Query object + * e.g. "?query=will%20smith" + * @param finalQuery expected final query from result.getQuery() + * e.g. "query 'AND will smith'" + * @param intentModel IntentModel to be added to the Query + */ + public void assertRewrittenQuery(String origQuery, String finalQuery, IntentModel intentModel) { + Query query = new Query(origQuery); + intentModel.setTo(query); + Result result = execution.search(query); + Assert.assertEquals(finalQuery, result.getQuery().toString()); + } + + /** + * Create a new interpretation with modification that + * contains the passed in query and score + * @param spellRewrite query to be used as modification + * @param score score to be used as modification score + * @param isQSSRW whether the modification is qss_rw + * @param isQSSSugg whether the modification is qss_sugg + * @return newly created interpretation with modification + */ + public Interpretation createInterpretation(String spellRewrite, double score, + boolean isQSSRW, boolean isQSSSugg) { + Modification modification = new Modification(spellRewrite); + Annotations annotation = modification.getAnnotation(); + annotation.put("score", score); + if(isQSSRW) + annotation.put("qss_rw", true); + if(isQSSSugg) + annotation.put("qss_sugg", true); + Interpretation interpretation = new Interpretation(modification); + return interpretation; + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/SearchChainDispatcherSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/SearchChainDispatcherSearcherTestCase.java new file mode 100644 index 00000000000..608706605f3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/SearchChainDispatcherSearcherTestCase.java @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.rewrite.test; + +import java.util.*; +import java.io.File; + +import com.yahoo.search.*; +import com.yahoo.search.searchchain.*; +import com.yahoo.search.query.rewrite.*; +import com.yahoo.search.query.rewrite.rewriters.*; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.query.rewrite.RewritesConfig; +import com.yahoo.search.intent.model.*; +import com.yahoo.component.chain.Chain; + +/** + * Test Cases for SearchChainDispatcherSearcher + * + * @author karenlee@yahoo-inc.com + */ +public class SearchChainDispatcherSearcherTestCase extends junit.framework.TestCase { + + private QueryRewriteSearcherTestUtils utils; + private final String NAME_REWRITER_CONFIG_PATH = "file:src/test/java/com/yahoo/search/query/rewrite/test/" + + "test_name_rewriter.cfg"; + private final String NAME_ENTITY_EXPAND_DICT_PATH = "src/test/java/com/yahoo/search/query/rewrite/test/" + + "name_rewriter_entity.fsa"; + private final String NAME_REWRITER_NAME = NameRewriter.REWRITER_NAME; + private final String MISSPELL_REWRITER_NAME = MisspellRewriter.REWRITER_NAME; + private final String US_MARKET_SEARCH_CHAIN = "us_qrw"; + + + /** + * Load the QueryRewriteSearcher and prepare the + * execution object + */ + @SuppressWarnings("deprecation") + protected void setUp() { + // Instantiate Name Rewriter + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(NAME_REWRITER_CONFIG_PATH); + HashMap<String, File> fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(NAME_ENTITY_EXPAND_DICT_PATH)); + NameRewriter nameRewriter = new NameRewriter(config, fileList); + + // Instantiate Misspell Rewriter + MisspellRewriter misspellRewriter = new MisspellRewriter(); + + // Create market search chain of two rewriters + ArrayList<Searcher> searchers = new ArrayList<>(); + searchers.add(misspellRewriter); + searchers.add(nameRewriter); + Chain<Searcher> marketSearchChain = new Chain<>(US_MARKET_SEARCH_CHAIN, searchers); + + // Add market search chain to the registry + SearchChainRegistry registry = new SearchChainRegistry(); + registry.register(marketSearchChain); + + // Instantiate Search Chain Dispatcher Searcher + SearchChainDispatcherSearcher searchChainDispatcher = new SearchChainDispatcherSearcher(); + + // Create a chain containing only the dispatcher + Chain<Searcher> mainSearchChain = new Chain<>(searchChainDispatcher); + Execution execution = new Execution(mainSearchChain, Execution.Context.createContextStub(registry, null)); + utils = new QueryRewriteSearcherTestUtils(execution); + } + + public SearchChainDispatcherSearcherTestCase(String name) { + super(name); + } + + /** + * Execute the market chain + * Query will be rewritten twice + */ + public void testMarketChain() { + IntentModel intentModel = new IntentModel( + utils.createInterpretation("wills smith", 0.9, + true, false), + utils.createInterpretation("will smith", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&QRWChain=" + US_MARKET_SEARCH_CHAIN + "&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + "=true", + "query 'OR (AND willl smith) (AND will smith) " + + "\"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'", + intentModel); + } + + /** + * Market chain is not valid + * Query will be passed to next rewriter + */ + public void testInvalidMarketChain() { + utils.assertRewrittenQuery("?query=will smith&QRWChain=abc&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true", + "query 'AND will smith'"); + } + + /** + * Empty market chain value + * Query will be passed to next rewriter + */ + public void testEmptyMarketChain() { + utils.assertRewrittenQuery("?query=will smith&QRWChain=&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true", + "query 'AND will smith'"); + } + + /** + * Searchers down the chain after SearchChainDispatcher + * should be executed + */ + @SuppressWarnings("deprecation") + public void testChainContinuation() { + // Instantiate Name Rewriter + RewritesConfig config = QueryRewriteSearcherTestUtils.createConfigObj(NAME_REWRITER_CONFIG_PATH); + HashMap<String, File> fileList = new HashMap<>(); + fileList.put(NameRewriter.NAME_ENTITY_EXPAND_DICT, new File(NAME_ENTITY_EXPAND_DICT_PATH)); + NameRewriter nameRewriter = new NameRewriter(config, fileList); + + // Instantiate Misspell Rewriter + MisspellRewriter misspellRewriter = new MisspellRewriter(); + + // Create market search chain of only misspell rewriter + Chain<Searcher> marketSearchChain = new Chain<>(US_MARKET_SEARCH_CHAIN, misspellRewriter); + + // Add market search chain to the registry + SearchChainRegistry registry = new SearchChainRegistry(); + registry.register(marketSearchChain); + + // Instantiate Search Chain Dispatcher Searcher + SearchChainDispatcherSearcher searchChainDispatcher = new SearchChainDispatcherSearcher(); + + // Create a chain containing the dispatcher and the name rewriter + ArrayList<Searcher> searchers = new ArrayList<>(); + searchers.add(searchChainDispatcher); + searchers.add(nameRewriter); + + // Create a chain containing only the dispatcher + Chain<Searcher> mainSearchChain = new Chain<>(searchers); + Execution execution = new Execution(mainSearchChain, Execution.Context.createContextStub(registry, null)); + new QueryRewriteSearcherTestUtils(execution); + + IntentModel intentModel = new IntentModel( + utils.createInterpretation("wills smith", 0.9, + true, false), + utils.createInterpretation("will smith", 1.0, + false, true)); + + utils.assertRewrittenQuery("?query=willl+smith&QRWChain=" + US_MARKET_SEARCH_CHAIN + "&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_RW + "=true&" + + MISSPELL_REWRITER_NAME + "." + RewriterConstants.QSS_SUGG + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.REWRITES_AS_UNIT_EQUIV + + "=true&" + + NAME_REWRITER_NAME + "." + RewriterConstants.ORIGINAL_AS_UNIT_EQUIV + "=true", + "query 'OR (AND willl smith) (AND will smith) " + + "\"will smith\" \"will smith movies\" " + + "\"will smith news\" \"will smith imdb\" " + + "\"will smith lyrics\" \"will smith dead\" " + + "\"will smith nfl\" \"will smith new movie hancock\" " + + "\"will smith biography\"'", + intentModel); + } +} + diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/generic_expansion.fsa b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/generic_expansion.fsa Binary files differnew file mode 100644 index 00000000000..2fb1db0cde2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/generic_expansion.fsa diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/name_rewriter_entity.fsa b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/name_rewriter_entity.fsa Binary files differnew file mode 100644 index 00000000000..0507632d2d9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/name_rewriter_entity.fsa diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_generic_expansion_rewriter.cfg b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_generic_expansion_rewriter.cfg new file mode 100644 index 00000000000..7b86c1385d7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_generic_expansion_rewriter.cfg @@ -0,0 +1,3 @@ +fsaDict[1] +fsaDict[0].name GenericExpansion +fsaDict[0].path dictionaries/GenericExpansionRewriter.fsa diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_name_rewriter.cfg b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_name_rewriter.cfg new file mode 100644 index 00000000000..ef0b82eb64d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_name_rewriter.cfg @@ -0,0 +1,3 @@ +fsaDict[1] +fsaDict[0].name NameEntityExpansion +fsaDict[0].path dictionaries/NameRewriter.fsa diff --git a/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_rewriter_fake_fsa.cfg b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_rewriter_fake_fsa.cfg new file mode 100644 index 00000000000..75cba5f8c90 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/rewrite/test/test_rewriter_fake_fsa.cfg @@ -0,0 +1,3 @@ +fsaDict[1] +fsaDict[0].name NameEntityExpansion +fsaDict[0].path dummyFSAPath diff --git a/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java new file mode 100644 index 00000000000..301438a5e41 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/ModelTestCase.java @@ -0,0 +1,112 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.prelude.query.Item; +import com.yahoo.search.Query; +import com.yahoo.search.query.Model; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.Arrays; +import java.util.LinkedHashSet; + + +/** + * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + */ +public class ModelTestCase extends junit.framework.TestCase { + + String oldConfigId; + + public void setUp() { + oldConfigId = System.getProperty("config.id"); + System.setProperty("config.id", "file:src/test/java/com/yahoo/prelude/test/fieldtypes/field-info.cfg"); + } + + + public void tearDown() { + if (oldConfigId == null) + System.getProperties().remove("config.id"); + else + System.setProperty("config.id", oldConfigId); + } + + public void testCopyParameters() { + Query q1 = new Query("?query=test1&filter=test2&defidx=content&default-index=lala&encoding=iso8859-1"); + Query q2 = q1.clone(); + Model r1 = q1.getModel(); + Model r2 = q2.getModel(); + assertTrue(r1 != r2); + assertEquals(r1,r2); + assertEquals("test1",r2.getQueryString()); + } + + public void testSetQuery() { + Query q1 = new Query("?query=test1"); + Item r1 = q1.getModel().getQueryTree(); + q1.properties().set("query","test2"); + q1.getModel().setQueryString(q1.getModel().getQueryString()); // Force reparse + assertNotSame(r1,q1.getModel().getQueryTree()); + q1.properties().set("query","test1"); + q1.getModel().setQueryString(q1.getModel().getQueryString()); // Force reparse + assertEquals(r1,q1.getModel().getQueryTree()); + } + + + public void testSetofSetters() { + Query q1 = new Query("?query=test1&encoding=iso-8859-1&language=en&default-index=subject&filter=" + enc("\u00C5")); + Model r1 = q1.getModel(); + assertEquals(r1.getQueryString(), "test1"); + assertEquals("iso-8859-1", r1.getEncoding()); + assertEquals("\u00C5", r1.getFilter()); + assertEquals("subject", r1.getDefaultIndex()); + } + + private String enc(String s) { + try { + return URLEncoder.encode(s, "utf-8"); + } + catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + + public void testSearchPath() { + assertEquals("c6/r8",new Query("?query=test1&model.searchPath=c6/r8").getModel().getSearchPath()); + assertEquals("c6/r8",new Query("?query=test1&searchpath=c6/r8").getModel().getSearchPath()); + } + + public void testClone() { + Query q= new Query(); + Model sr = new Model(q); + sr.setRestrict("music, cheese,other"); + sr.setSources("cluster1"); + assertEquals(sr.getSources(), new LinkedHashSet<>(Arrays.asList(new String[]{"cluster1"}))); + assertEquals(sr.getRestrict(),new LinkedHashSet<>(Arrays.asList(new String[]{"cheese","music","other"}))); + } + + public void testEquals() { + Query q = new Query(); + Model sra = new Model(q); + sra.setRestrict("music,cheese"); + sra.setSources("cluster1,cluster2"); + + Model srb = new Model(q); + srb.setRestrict(" cheese , music"); + srb.setSources("cluster1,cluster2"); + assertEquals(sra,srb); + srb.setRestrict("music,cheese"); + assertNotSame(sra,srb); + } + + public void testSearchRestrictQueryParameters() { + Query query=new Query("?query=test&search=news,archive&restrict=fish,bird"); + assertTrue(query.getModel().getSources().contains("news")); + assertTrue(query.getModel().getSources().contains("archive")); + assertEquals(2,query.getModel().getSources().size()); + assertTrue(query.getModel().getRestrict().contains("fish")); + assertTrue(query.getModel().getRestrict().contains("bird")); + assertEquals(2,query.getModel().getRestrict().size()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java new file mode 100644 index 00000000000..2f304693d1c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/ParametersTestCase.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.search.Query; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; + +import static com.yahoo.jdisc.http.HttpRequest.Method; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ParametersTestCase extends junit.framework.TestCase { + + public void testSettingRankProperty() { + Query query=new Query("?query=test&ranking.properties.dotProduct.X=(a:1,b:2)"); + assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + } + + public void testSettingRankPropertyAsAlias() { + Query query=new Query("?query=test&rankproperty.dotProduct.X=(a:1,b:2)"); + assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + } + + public void testSettingRankFeature() { + Query query=new Query("?query=test&ranking.features.matches=3"); + assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + } + + public void testSettingRankFeatureAsAlias() { + Query query=new Query("?query=test&rankfeature.matches=3"); + assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + } + + public void testSettingRankPropertyWithQueryProfile() { + Query query=new Query(HttpRequest.createTestRequest("?query=test&ranking.properties.dotProduct.X=(a:1,b:2)", Method.GET), createProfile()); + assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + } + + public void testSettingRankPropertyAsAliasWithQueryProfile() { + Query query=new Query(HttpRequest.createTestRequest("?query=test&rankproperty.dotProduct.X=(a:1,b:2)", Method.GET), createProfile()); + assertEquals("[(a:1,b:2)]",query.getRanking().getProperties().get("dotProduct.X").toString()); + } + + public void testSettingRankFeatureWithQueryProfile() { + Query query=new Query(HttpRequest.createTestRequest("?query=test&ranking.features.matches=3", Method.GET), createProfile()); + assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + } + + public void testSettingRankFeatureAsAliasWithQueryProfile() { + Query query=new Query(HttpRequest.createTestRequest("?query=test&rankfeature.matches=3", Method.GET), createProfile()); + assertEquals("3",query.getRanking().getFeatures().get("matches").toString()); + } + + public CompiledQueryProfile createProfile() { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("test"); + profile.set("model.filter", "+year:2001", registry); + profile.set("model.language", "en", registry); + return registry.compile().findQueryProfile("test"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/test/PresentationTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/PresentationTestCase.java new file mode 100644 index 00000000000..3222bd99cd9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/PresentationTestCase.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.prelude.query.Highlight; +import com.yahoo.search.Query; +import com.yahoo.search.query.Presentation; + +/** + * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + */ +public class PresentationTestCase extends junit.framework.TestCase { + + + public void testClone() { + Query q= new Query(""); + Presentation p = new Presentation(q); + p.setBolding(true); + Highlight h = new Highlight(); + h.addHighlightTerm("date","today"); + p.setHighlight(h); + Presentation pc = (Presentation)p.clone(); + h.addHighlightTerm("title","Hello"); + assertTrue(pc.getBolding()); + pc.getHighlight().getHighlightItems(); + assertTrue(pc.getHighlight().getHighlightItems().containsKey("date")); + assertFalse(pc.getHighlight().getHighlightItems().containsKey("title")); + + assertEquals(p,pc); + + } + + + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/test/QueryCloneMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/test/QueryCloneMicroBenchmark.java new file mode 100644 index 00000000000..7a9f5ce59b6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/QueryCloneMicroBenchmark.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.search.Query; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class QueryCloneMicroBenchmark { + + public void benchmark() { + int runs = 10000; + + Query query = createQuery(); + for (int i = 0; i<100000; i++) // yes, this much is needed + query.clone(); + long startTime = System.currentTimeMillis(); + for (int i = 0; i<runs; i++) + query.clone(); + long totalTime = System.currentTimeMillis() - startTime; + System.out.println("Time per clone: " + (totalTime * 1000 * 1000 / runs) + " nanoseconds" ); + } + + private Query createQuery() { + Query query = new Query(); + query.getModel().getQueryTree().setRoot(createWeightedSet()); + return query; + } + + private WeightedSetItem createWeightedSet() { + WeightedSetItem item = new WeightedSetItem("w"); + for (int i = 0; i<1000; i++) + item.addToken("item" + i, i); + return item; + } + + public static void main(String[] args) { + new QueryCloneMicroBenchmark().benchmark(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java new file mode 100644 index 00000000000..4ee4932bb65 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/test/RankingTestCase.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.test; + +import com.yahoo.search.Query; +import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.Sorting; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + */ +public class RankingTestCase extends junit.framework.TestCase { + + /** tests setting rank feature values */ + public void testRankFeatures() { + // Check initializing from query + Query query = new Query("?query=test&ranking.features.query(name)=0.1&ranking.features.fieldMatch(foo)=0.2"); + assertEquals("0.1", query.getRanking().getFeatures().get("query(name)")); + assertEquals("0.2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + assertEquals("{\"query(name)\":\"0.1\",\"fieldMatch(foo)\":\"0.2\"}", query.getRanking().getFeatures().toString()); + + // Test cloning + Query clone = query.clone(); + assertEquals("0.1", query.getRanking().getFeatures().get("query(name)")); + assertEquals("0.2", query.getRanking().getFeatures().get("fieldMatch(foo)")); + + // Check programmatic setting + that the clone really has a separate object + assertFalse(clone.getRanking().getFeatures() == query.getRanking().getFeatures()); + clone.properties().set("ranking.features.query(name)","0.3"); + assertEquals("0.3", clone.getRanking().getFeatures().get("query(name)")); + assertEquals("0.1", query.getRanking().getFeatures().get("query(name)")); + + // Check getting + assertEquals("0.3",clone.properties().get("ranking.features.query(name)")); + + // Check map access + assertEquals(2, query.getRanking().getFeatures().asMap().size()); + assertEquals("0.2", query.getRanking().getFeatures().asMap().get("fieldMatch(foo)")); + query.getRanking().getFeatures().asMap().put("fieldMatch(foo)", "0.3"); + assertEquals("0.3", query.getRanking().getFeatures().get("fieldMatch(foo)")); + } + + //This test is order dependent. Fix this!! + public void test_setting_rank_feature_values() { + // Check initializing from query + Query query = new Query("?query=test&ranking.properties.foo=bar1&ranking.properties.foo2=bar2&ranking.properties.other=10"); + assertEquals("bar1", query.getRanking().getProperties().get("foo").get(0)); + assertEquals("bar2", query.getRanking().getProperties().get("foo2").get(0)); + assertEquals("10", query.getRanking().getProperties().get("other").get(0)); + assertEquals("{\"other\":[10],\"foo\":[bar1],\"foo2\":[bar2]}", query.getRanking().getProperties().toString()); + + // Test cloning + Query clone = query.clone(); + assertFalse(clone.getRanking().getProperties() == query.getRanking().getProperties()); + assertEquals("bar1", clone.getRanking().getProperties().get("foo").get(0)); + assertEquals("bar2", clone.getRanking().getProperties().get("foo2").get(0)); + assertEquals("10", clone.getRanking().getProperties().get("other").get(0)); + + // Check programmatic setting mean addition + clone.properties().set("ranking.properties.other","12"); + assertEquals("[10, 12]", clone.getRanking().getProperties().get("other").toString()); + assertEquals("[10]", query.getRanking().getProperties().get("other").toString()); + + // Check map access + assertEquals(3, query.getRanking().getProperties().asMap().size()); + assertEquals("bar1", query.getRanking().getProperties().asMap().get("foo").get(0)); + } + + /** Test setting sorting to null does not cause an exception. */ + public void testResetSorting() { + Query q=new Query(); + q.getRanking().setSorting((Sorting)null); + q.getRanking().setSorting((String)null); + } + + /** Tests deprecated naming */ + @Test + public void testFeatureOverride() { + Query query = new Query("?query=abc&featureoverride.something=2"); + assertEquals("2", query.getRanking().getFeatures().get("something")); + } + + @Test + public void testStructuredRankProperty() { + Query query = new Query("?query=abc&rankproperty.distanceToPath(gps_position).path=(0,0,10,0,10,5,20,5)"); + assertEquals("(0,0,10,0,10,5,20,5)", query.getRanking().getProperties().get("distanceToPath(gps_position).path").get(0).toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java b/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java new file mode 100644 index 00000000000..d59fd23e567 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java @@ -0,0 +1,175 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.textserialize.item.test; + +import com.yahoo.prelude.query.*; +import com.yahoo.search.query.textserialize.item.ItemContext; +import com.yahoo.search.query.textserialize.item.ItemFormHandler; +import com.yahoo.search.query.textserialize.parser.ParseException; +import com.yahoo.search.query.textserialize.parser.Parser; +import org.junit.Test; + +import java.io.StringReader; + +import static junit.framework.Assert.assertNull; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsInstanceOf.instanceOf; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class ParseItemTestCase { + public static Object parse(String input) throws ParseException { + ItemContext context = new ItemContext(); + Object result = new Parser(new StringReader(input.replace("'", "\"")), new ItemFormHandler(), context).start(); + context.connectItems(); + return result; + } + + @Test + public void parse_and() throws ParseException { + assertThat(parse("(AND)"), instanceOf(AndItem.class)); + } + + @Test + public void parse_and_with_children() throws ParseException { + AndItem andItem = (AndItem) parse("(AND (WORD 'first') (WORD 'second'))"); + + assertThat(andItem.getItemCount(), is(2)); + assertThat(getWord(andItem.getItem(0)), is("first")); + } + + @Test + public void parse_or() throws ParseException { + assertThat(parse("(OR)"), instanceOf(OrItem.class)); + } + + @Test + public void parse_and_not_rest() throws ParseException { + assertThat(parse("(AND-NOT-REST)"), instanceOf(NotItem.class)); + } + + @Test + public void parse_and_not_rest_with_children() throws ParseException { + NotItem notItem = (NotItem) parse("(AND-NOT-REST (WORD 'positive') (WORD 'negative'))"); + assertThat(getWord(notItem.getPositiveItem()), is("positive")); + assertThat(getWord(notItem.getItem(1)), is("negative")); + } + + @Test + public void parse_and_not_rest_with_only_negated_children() throws ParseException { + NotItem notItem = (NotItem) parse("(AND-NOT-REST null (WORD 'negated-item'))"); + assertNull(notItem.getPositiveItem()); + assertThat(notItem.getItem(1), instanceOf(WordItem.class)); + } + + @Test + public void parse_rank() throws ParseException { + assertThat(parse("(RANK (WORD 'first'))"), instanceOf(RankItem.class)); + } + + @Test + public void parse_word() throws ParseException { + WordItem wordItem = (WordItem) parse("(WORD 'text')"); + assertThat(wordItem.getWord(), is("text")); + } + + @Test(expected = IllegalArgumentException.class) + public void fail_when_word_given_multiple_strings() throws ParseException { + parse("(WORD 'one' 'invalid')"); + } + + @Test(expected = IllegalArgumentException.class) + public void fail_when_word_given_no_string() throws ParseException { + parse("(WORD)"); + } + + @Test + public void parse_int() throws ParseException { + IntItem intItem = (IntItem) parse("(INT '[42;]')"); + assertThat(intItem.getNumber(), is("[42;]")); + } + + @Test + public void parse_range() throws ParseException { + IntItem intItem = (IntItem) parse("(INT '[42;73]')"); + assertThat(intItem.getNumber(), is("[42;73]")); + } + + @Test + public void parse_range_withlimit() throws ParseException { + IntItem intItem = (IntItem) parse("(INT '[42;73;32]')"); + assertThat(intItem.getNumber(), is("[42;73;32]")); + } + + @Test + public void parse_prefix() throws ParseException { + PrefixItem prefixItem = (PrefixItem) parse("(PREFIX 'word')"); + assertThat(prefixItem.getWord(), is("word")); + } + + @Test + public void parse_subString() throws ParseException { + SubstringItem subStringItem = (SubstringItem) parse("(SUBSTRING 'word')"); + assertThat(subStringItem.getWord(), is("word")); + } + + @Test + public void parse_exactString() throws ParseException { + ExactstringItem subStringItem = (ExactstringItem) parse("(EXACT 'word')"); + assertThat(subStringItem.getWord(), is("word")); + } + + @Test + public void parse_suffix() throws ParseException { + SuffixItem suffixItem = (SuffixItem) parse("(SUFFIX 'word')"); + assertThat(suffixItem.getWord(), is("word")); + } + + @Test + public void parse_phrase() throws ParseException { + PhraseItem phraseItem = (PhraseItem) parse("(PHRASE (WORD 'word'))"); + assertThat(phraseItem.getItem(0), instanceOf(WordItem.class)); + } + + @Test + public void parse_near() throws ParseException { + assertThat(parse("(NEAR)"), instanceOf(NearItem.class)); + } + + @Test + public void parse_onear() throws ParseException { + assertThat(parse("(ONEAR)"), instanceOf(ONearItem.class)); + } + + @Test + public void parse_near_with_distance() throws ParseException { + NearItem nearItem = (NearItem) parse("(NEAR {'distance' 42} (WORD 'first'))"); + assertThat(nearItem.getDistance(), is(42)); + } + + @Test + public void parse_items_with_connectivity() throws ParseException { + AndItem andItem = (AndItem) parse("(AND (WORD {'id' '1'} 'first') (WORD {'connectivity' ['1' 23.5]} 'second'))"); + WordItem secondItem = (WordItem) andItem.getItem(1); + + assertThat(secondItem.getConnectedItem(), is(andItem.getItem(0))); + assertThat(secondItem.getConnectivity(), is(23.5)); + } + + @Test + public void parse_word_with_index() throws ParseException { + WordItem wordItem = (WordItem) parse("(WORD {'index' 'someIndex'} 'text')"); + assertThat(wordItem.getIndexName(), is("someIndex")); + } + + @Test + public void parse_unicode_word() throws ParseException { + WordItem wordItem = (WordItem) parse("(WORD 'trăm')"); + assertThat(wordItem.getWord(), is("trăm")); + } + + public static String getWord(Object item) { + return ((WordItem)item).getWord(); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/query/textserialize/serializer/test/SerializeItemTestCase.java b/container-search/src/test/java/com/yahoo/search/query/textserialize/serializer/test/SerializeItemTestCase.java new file mode 100644 index 00000000000..47bf4072f60 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/textserialize/serializer/test/SerializeItemTestCase.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query.textserialize.serializer.test; + +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.query.textserialize.parser.ParseException; +import com.yahoo.search.query.textserialize.serializer.QueryTreeSerializer; +import org.junit.Test; + +import static com.yahoo.search.query.textserialize.item.test.ParseItemTestCase.parse; +import static com.yahoo.search.query.textserialize.item.test.ParseItemTestCase.getWord; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsInstanceOf.instanceOf; +import static org.junit.Assert.assertThat; + +/** + * @author tonytv + */ +public class SerializeItemTestCase { + @Test + public void serialize_word_item() { + WordItem item = new WordItem("test that \" and \\ works"); + item.setIndexName("index\"Name"); + + WordItem deSerialized = serializeThenParse(item); + assertThat(deSerialized.getWord(), is(item.getWord())); + assertThat(deSerialized.getIndexName(), is(item.getIndexName())); + } + + @Test + public void serialize_and_item() throws ParseException { + AndItem andItem = new AndItem(); + andItem.addItem(new WordItem("first")); + andItem.addItem(new WordItem("second")); + + AndItem deSerialized = serializeThenParse(andItem); + assertThat(getWord(deSerialized.getItem(0)), is("first")); + assertThat(getWord(deSerialized.getItem(1)), is("second")); + assertThat(deSerialized.getItemCount(), is(2)); + } + + @Test + public void serialize_or_item() throws ParseException { + assertThat(serializeThenParse(new OrItem()), + instanceOf(OrItem.class)); + } + + @Test + public void serialize_not_item() throws ParseException { + NotItem notItem = new NotItem(); + { + notItem.addItem(new WordItem("first")); + notItem.addItem(new WordItem("second")); + } + + serializeThenParse(notItem); + } + + @Test + public void serialize_near_item() throws ParseException { + int distance = 23; + NearItem nearItem = new NearItem(distance); + { + nearItem.addItem(new WordItem("first")); + nearItem.addItem(new WordItem("second")); + } + + NearItem deSerialized = serializeThenParse(nearItem); + + assertThat(deSerialized.getDistance(), is(distance)); + assertThat(deSerialized.getItemCount(), is(2)); + } + + @Test + public void serialize_phrase_item() throws ParseException { + PhraseItem phraseItem = new PhraseItem(new String[] {"first", "second"}); + phraseItem.setIndexName("indexName"); + + PhraseItem deSerialized = serializeThenParse(phraseItem); + assertThat(deSerialized.getItem(0), is(phraseItem.getItem(0))); + assertThat(deSerialized.getItem(1), is(phraseItem.getItem(1))); + assertThat(deSerialized.getIndexName(), is(phraseItem.getIndexName())); + } + + @Test + public void serialize_equiv_item() throws ParseException { + EquivItem equivItem = new EquivItem(); + equivItem.addItem(new WordItem("first")); + + EquivItem deSerialized = serializeThenParse(equivItem); + assertThat(deSerialized.getItemCount(), is(1)); + } + + @Test + public void serialize_connectivity() throws ParseException { + OrItem orItem = new OrItem(); + { + WordItem first = new WordItem("first"); + WordItem second = new WordItem("second"); + first.setConnectivity(second, 3.14); + + orItem.addItem(first); + orItem.addItem(second); + } + + OrItem deSerialized = serializeThenParse(orItem); + WordItem first = (WordItem) deSerialized.getItem(0); + Item second = deSerialized.getItem(1); + + assertThat(first.getConnectedItem(), is(second)); + assertThat(first.getConnectivity(), is(3.14)); + } + + @Test + public void serialize_significance() throws ParseException { + EquivItem equivItem = new EquivItem(); + equivItem.setSignificance(24.2); + + EquivItem deSerialized = serializeThenParse(equivItem); + assertThat(deSerialized.getSignificance(), is(24.2)); + } + + @Test + public void serialize_unique_id() throws ParseException { + EquivItem equivItem = new EquivItem(); + equivItem.setUniqueID(42); + + EquivItem deSerialized = serializeThenParse(equivItem); + assertThat(deSerialized.getUniqueID(), is(42)); + } + + @Test + public void serialize_weight() throws ParseException { + EquivItem equivItem = new EquivItem(); + equivItem.setWeight(42); + + EquivItem deSerialized = serializeThenParse(equivItem); + assertThat(deSerialized.getWeight(), is(42)); + } + + private static String serialize(Item item) { + return new QueryTreeSerializer().serialize(item); + } + + @SuppressWarnings("unchecked") + private static <T extends Item> T serializeThenParse(T oldItem) { + try { + return (T) parse(serialize(oldItem)); + } catch (ParseException e) { + throw new RuntimeException(e); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/BooleanAttributeParserTest.java b/container-search/src/test/java/com/yahoo/search/querytransform/BooleanAttributeParserTest.java new file mode 100644 index 00000000000..764a44a1bd6 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/BooleanAttributeParserTest.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import com.yahoo.prelude.query.PredicateQueryItem; +import org.junit.Test; + +import java.math.BigInteger; +import java.util.Iterator; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.fail; + +/** + * Created with IntelliJ IDEA. + * User: magnarn + * Date: 2/5/13 + * Time: 3:52 PM + */ +public class BooleanAttributeParserTest { + + @Test + public void requireThatParseHandlesAllFormats() throws Exception { + assertParse(null, 0); + assertParse("{}", 0); + assertParse("{foo:bar}", 1); + assertParse("{foo:[bar]}", 1); + assertParse("{foo:bar, baz:qux}", 2); + + assertParse("{foo:bar, foo:baz}", 2); + assertParse("{foo:[bar, baz, qux]}", 3); + assertParse("{foo:[bar, baz, qux], quux:corge}", 4); + assertParse("{foo:[bar, baz, qux], quux:[corge, grault]}", 5); + assertParse("{foo:bar, foo:bar, foo:bar}", 3); + + assertParse("{foo:bar:0x1, foo:baz:0xf}", 2); + assertParse("{foo:[bar:0xbabe, baz:0xbeef, qux:0xfee1], quux:corge:0x1234}", 4); + assertParse("{foo:bar:[1], foo:baz:[0,1,2,3]}", 2); + assertParse("{foo:bar:[ 1 ], foo:baz:[ 0 , 1 , 2 , 3 ]}", 2); + assertParse("{foo:[bar:[4,7],baz:[8,5],qux:[3,2]], quux:corge:[2, 5, 7, 58]}", 4); + } + + @Test + public void requireThatIllegalStringsFail() throws Exception { + assertException("{foo:[bar:[baz]}"); + assertException("{foo:[bar:baz}"); + assertException("{foo:bar:[0,1,2}"); + assertException("{foo:[bar:[0,1,2],baz:[0,,2]]}"); + assertException("{foo:[bar:[0,1,2],baz:[0,1,2]}"); + assertException("{foo:bar:[64]}"); + assertException("{foo:bar:[-1]}"); + assertException("{foo:bar:[a]}"); + assertException("{foo:bar:[0,1,[2]]}"); + assertException("{foo:bar}extrachars"); + } + + private void assertException(String s) { + try { + PredicateQueryItem item = new PredicateQueryItem(); + new BooleanSearcher.PredicateValueAttributeParser(item).parse(s); + fail("Expected an exception"); + } catch (IllegalArgumentException e) { + } + } + + @Test + public void requireThatTermsCanHaveBitmaps() throws Exception { + PredicateQueryItem q = assertParse("{foo:bar:0x1}", 1); + PredicateQueryItem.Entry[] features = new PredicateQueryItem.Entry[q.getFeatures().size()]; + q.getFeatures().toArray(features); + assertEquals(1l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:0x1, baz:qux:0xf}", 2); + Iterator<PredicateQueryItem.Entry> it = q.getFeatures().iterator(); + assertEquals(1l, it.next().getSubQueryBitmap()); + assertEquals(15l, it.next().getSubQueryBitmap()); + q = assertParse("{foo:bar:0xffffffffffffffff}", 1); + assertEquals(-1l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:[63]}", 1); + + assertEquals(new BigInteger("ffffffffffffffff", 16).shiftRight(1).add(BigInteger.ONE).longValue(), q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:0x7fffffffffffffff}", 1); + assertEquals(new BigInteger("ffffffffffffffff", 16).shiftRight(1).longValue(), q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:[0]}", 1); + assertEquals(1l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}", 1); + assertEquals(1l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:bar:[0,2,6,8]}", 1); + assertEquals(0x145l, q.getFeatures().iterator().next().getSubQueryBitmap()); + q = assertParse("{foo:[bar:[0,8,6,2],baz:[1,3,4,15]]}", 2); + it = q.getFeatures().iterator(); + assertEquals(0x145l, it.next().getSubQueryBitmap()); + assertEquals(0x801al, it.next().getSubQueryBitmap()); + } + + private PredicateQueryItem assertParse(String s, int numFeatures) { + PredicateQueryItem item = new PredicateQueryItem(); + BooleanAttributeParser parser = new BooleanSearcher.PredicateValueAttributeParser(item); + parser.parse(s); + assertEquals(numFeatures, item.getFeatures().size()); + return item; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/BooleanSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/BooleanSearcherTestCase.java new file mode 100644 index 00000000000..5c3d9b4824f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/BooleanSearcherTestCase.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collection; + +import static junit.framework.Assert.assertEquals; + +/** + * Test BooleanSearcher + */ +public class BooleanSearcherTestCase { + + private Execution exec; + + private Execution buildExec() { + return new Execution(new Chain<Searcher>(new BooleanSearcher()), + Execution.Context.createContextStub()); + } + + @Before + public void setUp() throws Exception { + exec = buildExec(); + } + + @Test + public void requireThatAttributeMapToSingleFeature() { + PredicateQueryItem item = buildPredicateQueryItem("{gender:female}", null); + assertEquals(1, item.getFeatures().size()); + assertEquals(0, item.getRangeFeatures().size()); + assertEquals("gender", item.getFeatures().iterator().next().getKey()); + assertEquals("female", item.getFeatures().iterator().next().getValue()); + assertEquals("PREDICATE_QUERY_ITEM gender=female", item.toString()); + } + + @Test + public void requireThatAttributeListMapToMultipleFeatures() { + PredicateQueryItem item = buildPredicateQueryItem("{gender:[female,male]}", null); + assertEquals(2, item.getFeatures().size()); + assertEquals(0, item.getRangeFeatures().size()); + assertEquals("PREDICATE_QUERY_ITEM gender=female, gender=male", item.toString()); + } + + @Test + public void requireThatRangeAttributesMapToRangeTerm() { + PredicateQueryItem item = buildPredicateQueryItem(null, "{age:25}"); + assertEquals(0, item.getFeatures().size()); + assertEquals(1, item.getRangeFeatures().size()); + assertEquals("PREDICATE_QUERY_ITEM age:25", item.toString()); + + item = buildPredicateQueryItem(null, "{age:25:0x43, height:170:[2,3,4]}"); + assertEquals(0, item.getFeatures().size()); + assertEquals(2, item.getRangeFeatures().size()); + } + + @Test + public void requireThatQueryWithoutBooleanPropertiesIsUnchanged() { + Query q = new Query(""); + q.getModel().getQueryTree().setRoot(new WordItem("foo", "otherfield")); + Result r = exec.search(q); + + WordItem root = (WordItem)r.getQuery().getModel().getQueryTree().getRoot(); + assertEquals("foo", root.getWord()); + } + + @Test + public void requireThatBooleanSearcherCanBuildPredicateQueryItem() { + PredicateQueryItem root = buildPredicateQueryItem("{gender:female}", "{age:23:[2, 3, 5]}"); + + Collection<PredicateQueryItem.Entry> features = root.getFeatures(); + assertEquals(1, features.size()); + PredicateQueryItem.Entry entry = (PredicateQueryItem.Entry) features.toArray()[0]; + assertEquals("gender", entry.getKey()); + assertEquals("female", entry.getValue()); + assertEquals(-1L, entry.getSubQueryBitmap()); + + Collection<PredicateQueryItem.RangeEntry> rangeFeatures = root.getRangeFeatures(); + assertEquals(1, rangeFeatures.size()); + PredicateQueryItem.RangeEntry rangeEntry = (PredicateQueryItem.RangeEntry) rangeFeatures.toArray()[0]; + assertEquals("age", rangeEntry.getKey()); + assertEquals(23L, rangeEntry.getValue()); + assertEquals(44L, rangeEntry.getSubQueryBitmap()); + } + + @Test + public void requireThatKeysAndValuesCanContainSpaces() { + PredicateQueryItem item = buildPredicateQueryItem("{'My Key':'My Value'}", null); + assertEquals(1, item.getFeatures().size()); + assertEquals(0, item.getRangeFeatures().size()); + assertEquals("My Key", item.getFeatures().iterator().next().getKey()); + assertEquals("'My Value'", item.getFeatures().iterator().next().getValue()); + assertEquals("PREDICATE_QUERY_ITEM My Key='My Value'", item.toString()); + } + + private PredicateQueryItem buildPredicateQueryItem(String attributes, String rangeAttributes) { + Query q = buildQuery("predicate", attributes, rangeAttributes); + Result r = exec.search(q); + return (PredicateQueryItem)r.getQuery().getModel().getQueryTree().getRoot(); + } + + private Query buildQuery(String field, String attributes, String rangeAttributes) { + Query q = new Query(""); + q.properties().set("boolean.field", field); + if (attributes != null) { + q.properties().set("boolean.attributes", attributes); + } + if (rangeAttributes != null) { + q.properties().set("boolean.rangeAttributes", rangeAttributes); + } + return q; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java new file mode 100644 index 00000000000..77c9d1ddb97 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/LegacyCombinatorTestCase.java @@ -0,0 +1,245 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import junit.framework.TestCase; + +import com.yahoo.container.protect.Error; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Unit testing of the searcher com.yahoo.search.querytransform.LegacyCombinator. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class LegacyCombinatorTestCase extends TestCase { + Searcher searcher; + + protected void setUp() throws Exception { + searcher = new LegacyCombinator(); + } + + public void testStraightForwardSearch() { + Query q = new Query("?query=a&query.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a b", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a juhu.22[gnuff]:b", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&query.juhu="); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("a", q.getModel().getQueryTree().toString()); + q = new Query("?query=a+c&query.juhu=b"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a c b", q.getModel().getQueryTree().toString()); + } + + public void testNoBaseQuery() { + Query q = new Query("?query.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("b", q.getModel().getQueryTree().toString()); + } + + public void testIncompatibleNewAndOldQuery() { + Query q = new Query("?query.juhu=b&defidx.juhu=a&query.juhu.defidx=c"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("NULL", q.getModel().getQueryTree().toString()); + assertTrue("No expected error found.", q.errors().size() > 0); + assertEquals("Did not get invalid query parameter error as expected.", + Error.INVALID_QUERY_PARAMETER.code, q.errors().get(0).getCode()); + } + + public void testNotCombinatorWithoutRoot() { + Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=not"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("NULL", q.getModel().getQueryTree().toString()); + assertTrue("No expected error found.", q.errors().size() > 0); + System.out.println(q.errors()); + assertEquals("Did not get invalid query parameter error as expected.", + Error.INVALID_QUERY_PARAMETER.code, q.errors().get(0).getCode()); + } + + public void testRankCombinator() { + Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("nalle:b", q.getModel().getQueryTree().toString()); + } + + public void testRankAndNot() { + Query q = new Query("?query.yahoo=2&query.yahoo.defidx=1&query.yahoo.operator=not&query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("+nalle:b -1:2", q.getModel().getQueryTree().toString()); + } + + public void testReqAndRankAndNot() { + Query q = new Query("?query.yahoo=2&query.yahoo.defidx=1&query.yahoo.operator=not&query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("+(RANK y:z nalle:b) -1:2", q.getModel().getQueryTree().toString()); + } + + public void testReqAndRank() { + Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=rank&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("RANK y:z nalle:b", q.getModel().getQueryTree().toString()); + } + + public void testReqAndNot() { + Query q = new Query("?query.juhu=b&query.juhu.defidx=nalle&query.juhu.operator=not&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("+y:z -nalle:b", q.getModel().getQueryTree().toString()); + } + + public void testNewAndOld() { + Query q = new Query("?query.juhu=b&defidx.juhu=nalle&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + Set<StringPair> nastierItems = new HashSet<>(); + nastierItems.add(new StringPair("nalle", "b")); + nastierItems.add(new StringPair("y", "z")); + e.search(q); + AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); + Iterator<?> iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); + if (nastierItems.contains(asPair)) { + nastierItems.remove(asPair); + } else { + assertFalse("Got unexpected item in query tree: (" + + word.getIndexName() + ", " + word.stringValue() + ")", + true); + } + } + assertEquals("Not all expected items found in query.", 0, nastierItems.size()); + } + + public void testReqAndNotWithQuerySyntaxAll() { + Query q = new Query("?query.juhu=b+c&query.juhu.defidx=nalle&query.juhu.operator=not&query.juhu.type=any&query.bamse=z&query.bamse.defidx=y"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("+y:z -(OR nalle:b nalle:c)", q.getModel().getQueryTree().toString()); + } + + public void testDefaultIndexWithoutQuery() { + Query q = new Query("?defidx.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("NULL", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&defidx.juhu=b"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("a", q.getModel().getQueryTree().toString()); + } + + private static class StringPair { + public final String index; + public final String value; + + StringPair(String index, String value) { + super(); + this.index = index; + this.value = value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((index == null) ? 0 : index.hashCode()); + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final StringPair other = (StringPair) obj; + if (index == null) { + if (other.index != null) + return false; + } else if (!index.equals(other.index)) + return false; + if (value == null) { + if (other.value != null) + return false; + } else if (!value.equals(other.value)) + return false; + return true; + } + + } + + public void testMultiPart() { + Query q = new Query("?query=a&query.juhu=b&query.nalle=c"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + Set<String> items = new HashSet<>(); + items.add("a"); + items.add("b"); + items.add("c"); + e.search(q); + // OK, the problem here is we have no way of knowing whether nalle or + // juhu was added first, since we have passed through HashMap instances + // inside the implementation + + AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); + Iterator<?> iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + if (items.contains(word.stringValue())) { + items.remove(word.stringValue()); + } else { + assertFalse("Got unexpected item in query tree: " + word.stringValue(), true); + } + } + assertEquals("Not all expected items found in query.", 0, items.size()); + + Set<StringPair> nastierItems = new HashSet<>(); + nastierItems.add(new StringPair("", "a")); + nastierItems.add(new StringPair("juhu.22[gnuff]", "b")); + nastierItems.add(new StringPair("gnuff[8].name(\"tralala\")", "c")); + q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]&query.nalle=c&defidx.nalle=gnuff[8].name(%22tralala%22)"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + root = (AndItem) q.getModel().getQueryTree().getRoot(); + iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); + if (nastierItems.contains(asPair)) { + nastierItems.remove(asPair); + } else { + assertFalse("Got unexpected item in query tree: (" + + word.getIndexName() + ", " + word.stringValue() + ")", + true); + } + } + assertEquals("Not all expected items found in query.", 0, nastierItems.size()); + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/LowercasingTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/LowercasingTestCase.java new file mode 100644 index 00000000000..6b8f8861af5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/LowercasingTestCase.java @@ -0,0 +1,217 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordAlternativesItem.Alternative; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * Tests term lowercasing in the search chain. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class LowercasingTestCase { + + private static final String TEDDY = "teddy"; + private static final String BAMSE = "bamse"; + IndexFacts settings; + Execution execution; + + @Before + public void setUp() throws Exception { + IndexFacts f = new IndexFacts(); + Index bamse = new Index(BAMSE); + Index teddy = new Index(TEDDY); + Index defaultIndex = new Index("default"); + bamse.setLowercase(true); + teddy.setLowercase(false); + defaultIndex.setLowercase(true); + f.addIndex("nalle", bamse); + f.addIndex("nalle", teddy); + f.addIndex("nalle", defaultIndex); + f.freeze(); + settings = f; + execution = new Execution(new Chain<Searcher>( + new VespaLowercasingSearcher(new LowercasingConfig(new LowercasingConfig.Builder()))), + Execution.Context.createContextStub(settings)); + } + + @After + public void tearDown() throws Exception { + execution = null; + } + + @Test + public void smoke() { + Query q = new Query(); + AndItem root = new AndItem(); + WordItem tmp; + tmp = new WordItem("Gnuff", BAMSE, true); + root.addItem(tmp); + tmp = new WordItem("Blaff", TEDDY, true); + root.addItem(tmp); + tmp = new WordItem("Blyant", "", true); + root.addItem(tmp); + q.getModel().getQueryTree().setRoot(root); + + Result r = execution.search(q); + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WordItem w0 = (WordItem) root.getItem(0); + WordItem w1 = (WordItem) root.getItem(1); + WordItem w2 = (WordItem) root.getItem(2); + assertEquals("gnuff", w0.getWord()); + assertEquals("Blaff", w1.getWord()); + assertEquals("blyant", w2.getWord()); + } + + @Test + public void slightlyMoreComplexTree() { + Query q = new Query(); + AndItem a0 = new AndItem(); + OrItem o0 = new OrItem(); + PhraseItem p0 = new PhraseItem(); + p0.setIndexName(BAMSE); + PhraseSegmentItem p1 = new PhraseSegmentItem("Overbuljongterningpakkmesterassistent", true, false); + p1.setIndexName(BAMSE); + + WordItem tmp; + tmp = new WordItem("Nalle0", BAMSE, true); + a0.addItem(tmp); + + tmp = new WordItem("Nalle1", BAMSE, true); + o0.addItem(tmp); + tmp = new WordItem("Nalle2", BAMSE, true); + o0.addItem(tmp); + a0.addItem(o0); + + tmp = new WordItem("Nalle3", BAMSE, true); + p0.addItem(tmp); + + p1.addItem(new WordItem("Over", BAMSE, true)); + p1.addItem(new WordItem("buljong", BAMSE, true)); + p1.addItem(new WordItem("terning", BAMSE, true)); + p1.addItem(new WordItem("pakk", BAMSE, true)); + p1.addItem(new WordItem("Mester", BAMSE, true)); + p1.addItem(new WordItem("assistent", BAMSE, true)); + p1.lock(); + p0.addItem(p1); + a0.addItem(p0); + + q.getModel().getQueryTree().setRoot(a0); + + Result r = execution.search(q); + AndItem root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + tmp = (WordItem) root.getItem(0); + assertEquals("nalle0", tmp.getWord()); + OrItem orElement = (OrItem) root.getItem(1); + tmp = (WordItem) orElement.getItem(0); + assertEquals("nalle1", tmp.getWord()); + tmp = (WordItem) orElement.getItem(1); + assertEquals("nalle2", tmp.getWord()); + PhraseItem phrase = (PhraseItem) root.getItem(2); + tmp = (WordItem) phrase.getItem(0); + assertEquals("nalle3", tmp.getWord()); + PhraseSegmentItem locked = (PhraseSegmentItem) phrase.getItem(1); + assertEquals("over", ((WordItem) locked.getItem(0)).getWord()); + assertEquals("buljong", ((WordItem) locked.getItem(1)).getWord()); + assertEquals("terning", ((WordItem) locked.getItem(2)).getWord()); + assertEquals("pakk", ((WordItem) locked.getItem(3)).getWord()); + assertEquals("mester", ((WordItem) locked.getItem(4)).getWord()); + assertEquals("assistent", ((WordItem) locked.getItem(5)).getWord()); + } + + @Test + public void testWeightedSet() { + Query q = new Query(); + AndItem root = new AndItem(); + WeightedSetItem tmp; + tmp = new WeightedSetItem(BAMSE); + tmp.addToken("AbC", 3); + root.addItem(tmp); + tmp = new WeightedSetItem(TEDDY); + tmp.addToken("dEf", 5); + root.addItem(tmp); + q.getModel().getQueryTree().setRoot(root); + Result r = execution.search(q); + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WeightedSetItem w0 = (WeightedSetItem) root.getItem(0); + WeightedSetItem w1 = (WeightedSetItem) root.getItem(1); + assertEquals(1, w0.getNumTokens()); + assertEquals(1, w1.getNumTokens()); + assertEquals("abc", w0.getTokens().next().getKey()); + assertEquals("dEf", w1.getTokens().next().getKey()); + } + + @Test + public void testDisableLowercasingWeightedSet() { + execution = new Execution(new Chain<Searcher>( + new VespaLowercasingSearcher(new LowercasingConfig( + new LowercasingConfig.Builder() + .transform_weighted_sets(false)))), + Execution.Context.createContextStub(settings)); + + Query q = new Query(); + AndItem root = new AndItem(); + WeightedSetItem tmp; + tmp = new WeightedSetItem(BAMSE); + tmp.addToken("AbC", 3); + root.addItem(tmp); + tmp = new WeightedSetItem(TEDDY); + tmp.addToken("dEf", 5); + root.addItem(tmp); + q.getModel().getQueryTree().setRoot(root); + Result r = execution.search(q); + root = (AndItem) r.getQuery().getModel().getQueryTree().getRoot(); + WeightedSetItem w0 = (WeightedSetItem) root.getItem(0); + WeightedSetItem w1 = (WeightedSetItem) root.getItem(1); + assertEquals(1, w0.getNumTokens()); + assertEquals(1, w1.getNumTokens()); + assertEquals("AbC", w0.getTokens().next().getKey()); + assertEquals("dEf", w1.getTokens().next().getKey()); + } + + @Test + public void testLowercasingWordAlternatives() { + execution = new Execution(new Chain<Searcher>(new VespaLowercasingSearcher(new LowercasingConfig( + new LowercasingConfig.Builder().transform_weighted_sets(false)))), Execution.Context.createContextStub(settings)); + + Query q = new Query(); + WordAlternativesItem root; + List<WordAlternativesItem.Alternative> terms = new ArrayList<>(); + terms.add(new Alternative("ABC", 1.0)); + terms.add(new Alternative("def", 1.0)); + root = new WordAlternativesItem(BAMSE, true, null, terms); + q.getModel().getQueryTree().setRoot(root); + Result r = execution.search(q); + root = (WordAlternativesItem) r.getQuery().getModel().getQueryTree().getRoot(); + assertEquals(3, root.getAlternatives().size()); + assertEquals("ABC", root.getAlternatives().get(0).word); + assertEquals(1.0d, root.getAlternatives().get(0).exactness, 1e-15d); + assertEquals("abc", root.getAlternatives().get(1).word); + assertEquals(.7d, root.getAlternatives().get(1).exactness, 1e-15d); + assertEquals("def", root.getAlternatives().get(2).word); + assertEquals(1.0d, root.getAlternatives().get(2).exactness, 1e-15d); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java b/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java new file mode 100644 index 00000000000..512c28b28b1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import com.yahoo.prelude.query.Item; + +import com.yahoo.search.Result; + +public class TestUtils { + public static Item getQueryTreeRoot(Result result) { + return result.getQuery().getModel().getQueryTree().getRoot(); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/WandSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/WandSearcherTestCase.java new file mode 100644 index 00000000000..cbd168225d4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/WandSearcherTestCase.java @@ -0,0 +1,232 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.*; +import com.yahoo.processing.request.ErrorMessage; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import org.junit.Before; +import org.junit.Test; + +import java.util.ListIterator; + +import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertNotNull; + +/** + * Testing of WandSearcher. + */ +public class WandSearcherTestCase { + + private static final String VESPA_FIELD = "vespa-field"; + + private Execution exec; + + private IndexFacts buildIndexFacts() { + IndexFacts retval = new IndexFacts(); + retval.addIndex("test", new Index(VESPA_FIELD)); + retval.freeze(); + return retval; + } + + private Execution buildExec() { + return new Execution(new Chain<Searcher>(new WandSearcher()), + Execution.Context.createContextStub(buildIndexFacts())); + } + + private Query buildQuery(String wandFieldName, String wandTokens, String wandHeapSize, String wandType, String wandScoreThreshold, String wandThresholdBoostFactor) { + Query q = new Query(""); + q.properties().set("wand.field", wandFieldName); + q.properties().set("wand.tokens", wandTokens); + if (wandHeapSize != null) { + q.properties().set("wand.heapSize", wandHeapSize); + } + if (wandType != null) { + q.properties().set("wand.type", wandType); + } + if (wandScoreThreshold != null) { + q.properties().set("wand.scoreThreshold", wandScoreThreshold); + } + if (wandThresholdBoostFactor != null) { + q.properties().set("wand.thresholdBoostFactor", wandThresholdBoostFactor); + } + q.setHits(9); + return q; + } + + private Query buildDefaultQuery(String wandFieldName, String wandHeapSize) { + return buildQuery(wandFieldName, "{a:1,b:2,c:3}", wandHeapSize, null, null, null); + } + + private Query buildDefaultQuery() { + return buildQuery(VESPA_FIELD, "{a:1,\"b\":2,c:3}", null, null, null, null); + } + + private void assertWordItem(String expToken, String expField, int expWeight, Item item) { + WordItem wordItem = (WordItem) item; + assertEquals(expToken, wordItem.getWord()); + assertEquals(expField, wordItem.getIndexName()); + assertEquals(expWeight, wordItem.getWeight()); + } + + @Before + public void setUp() throws Exception { + exec = buildExec(); + } + + @Test + public void requireThatVespaWandCanBeSpecified() { + Query q = buildDefaultQuery(); + Result r = exec.search(q); + + WeakAndItem root = (WeakAndItem)TestUtils.getQueryTreeRoot(r); + assertEquals(100, root.getN()); + assertEquals(3, root.getItemCount()); + ListIterator<Item> itr = root.getItemIterator(); + assertWordItem("a", VESPA_FIELD, 1, itr.next()); + assertWordItem("b", VESPA_FIELD, 2, itr.next()); + assertWordItem("c", VESPA_FIELD, 3, itr.next()); + assertFalse(itr.hasNext()); + } + + @Test + public void requireThatVespaWandHeapSizeCanBeSpecified() { + Query q = buildDefaultQuery(VESPA_FIELD, "50"); + Result r = exec.search(q); + + WeakAndItem root = (WeakAndItem)TestUtils.getQueryTreeRoot(r); + assertEquals(50, root.getN()); + } + + @Test + public void requireThatWandCanBeSpecifiedTogetherWithNonAndQueryRoot() { + Query q = buildDefaultQuery(); + q.getModel().getQueryTree().setRoot(new WordItem("foo", "otherfield")); + Result r = exec.search(q); + + AndItem root = (AndItem)TestUtils.getQueryTreeRoot(r); + assertEquals(2, root.getItemCount()); + ListIterator<Item> itr = root.getItemIterator(); + assertTrue(itr.next() instanceof WordItem); + assertTrue(itr.next() instanceof WeakAndItem); + assertFalse(itr.hasNext()); + } + + @Test + public void requireThatWandCanBeSpecifiedTogetherWithAndQueryRoot() { + Query q = buildDefaultQuery(); + { + AndItem root = new AndItem(); + root.addItem(new WordItem("foo", "otherfield")); + root.addItem(new WordItem("bar", "otherfield")); + q.getModel().getQueryTree().setRoot(root); + } + Result r = exec.search(q); + + AndItem root = (AndItem)TestUtils.getQueryTreeRoot(r); + assertEquals(3, root.getItemCount()); + ListIterator<Item> itr = root.getItemIterator(); + assertTrue(itr.next() instanceof WordItem); + assertTrue(itr.next() instanceof WordItem); + assertTrue(itr.next() instanceof WeakAndItem); + assertFalse(itr.hasNext()); + } + + + @Test + public void requireThatNothingIsAddedWithoutWandPropertiesSet() { + Query foo = new Query(""); + foo.getModel().getQueryTree().setRoot(new WordItem("foo", "otherfield")); + Result r = exec.search(foo); + + WordItem root = (WordItem)TestUtils.getQueryTreeRoot(r); + assertEquals("foo", root.getWord()); + } + + @Test + public void requireThatErrorIsReturnedOnInvalidTokenList() { + Query q = buildQuery(VESPA_FIELD, "{a1,b:1}", null, null, null, null); + Result r = exec.search(q); + + ErrorMessage msg = r.hits().getError(); + assertNotNull(msg); + assertEquals(INVALID_QUERY_PARAMETER.code, msg.getCode()); + assertEquals("'{a1,b:1}' is not a legal sparse vector string: Expected ':' starting at position 3 but was ','",msg.getDetailedMessage()); + } + + @Test + public void requireThatErrorIsReturnedOnUnknownField() { + Query q = buildDefaultQuery("unknown", "50"); + Result r = exec.search(q); + ErrorMessage msg = r.hits().getError(); + assertNotNull(msg); + assertEquals(INVALID_QUERY_PARAMETER.code, msg.getCode()); + assertEquals("Field 'unknown' was not found in index facts for search definitions [test]",msg.getDetailedMessage()); + } + + @Test + public void requireThatVespaOrCanBeSpecified() { + Query q = buildQuery(VESPA_FIELD, "{a:1,b:2,c:3}", null, "or", null, null); + Result r = exec.search(q); + + OrItem root = (OrItem)TestUtils.getQueryTreeRoot(r); + assertEquals(3, root.getItemCount()); + ListIterator<Item> itr = root.getItemIterator(); + assertWordItem("a", VESPA_FIELD, 1, itr.next()); + assertWordItem("b", VESPA_FIELD, 2, itr.next()); + assertWordItem("c", VESPA_FIELD, 3, itr.next()); + assertFalse(itr.hasNext()); + } + + private void assertWeightedSetItem(WeightedSetItem item) { + assertEquals(3, item.getNumTokens()); + assertEquals(new Integer(1), item.getTokenWeight("a")); + assertEquals(new Integer(2), item.getTokenWeight("b")); + assertEquals(new Integer(3), item.getTokenWeight("c")); + } + + @Test + public void requireThatDefaultParallelWandCanBeSpecified() { + Query q = buildQuery(VESPA_FIELD, "{a:1,b:2,c:3}", null, "parallel", null, null); + Result r = exec.search(q); + + WandItem root = (WandItem)TestUtils.getQueryTreeRoot(r); + assertEquals(VESPA_FIELD, root.getIndexName()); + assertEquals(100, root.getTargetNumHits()); + assertEquals(0.0, root.getScoreThreshold()); + assertEquals(1.0, root.getThresholdBoostFactor()); + assertWeightedSetItem(root); + } + + @Test + public void requireThatParallelWandCanBeSpecified() { + Query q = buildQuery(VESPA_FIELD, "{a:1,b:2,c:3}", "50", "parallel", "70.5", "2.3"); + Result r = exec.search(q); + + WandItem root = (WandItem)TestUtils.getQueryTreeRoot(r); + assertEquals(VESPA_FIELD, root.getIndexName()); + assertEquals(50, root.getTargetNumHits()); + assertEquals(70.5, root.getScoreThreshold()); + assertEquals(2.3, root.getThresholdBoostFactor()); + assertWeightedSetItem(root); + } + + @Test + public void requireThatDotProductCanBeSpecified() { + Query q = buildQuery(VESPA_FIELD, "{a:1,b:2,c:3}", null, "dotProduct", null, null); + Result r = exec.search(q); + + DotProductItem root = (DotProductItem)TestUtils.getQueryTreeRoot(r); + assertEquals(VESPA_FIELD, root.getIndexName()); + assertWeightedSetItem(root); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java new file mode 100644 index 00000000000..4479650cd49 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java @@ -0,0 +1,369 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform.test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.prelude.hitfield.XMLString; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.querytransform.CJKSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.querytransform.NGramSearcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; + +import static com.yahoo.search.searchchain.Execution.Context.createContextStub; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class NGramSearcherTestCase extends junit.framework.TestCase { + + private Searcher searcher; + private IndexFacts indexFacts; + + @Override + public void setUp() { + searcher=new NGramSearcher(new SimpleLinguistics()); + indexFacts=new IndexFacts(); + + Index defaultIndex=new Index("default"); + defaultIndex.setNGram(true,3); + defaultIndex.setDynamicSummary(true); + indexFacts.addIndex("default",defaultIndex); + + Index test=new Index("test"); + test.setHighlightSummary(true); + indexFacts.addIndex("default",test); + + Index gram2=new Index("gram2"); + gram2.setNGram(true,2); + gram2.setDynamicSummary(true); + indexFacts.addIndex("default",gram2); + + Index gram3=new Index("gram3"); + gram3.setNGram(true,3); + gram3.setHighlightSummary(true); + indexFacts.addIndex("default",gram3); + + Index gram14=new Index("gram14"); + gram14.setNGram(true,14); + gram14.setDynamicSummary(true); + indexFacts.addIndex("default",gram14); + } + + private IndexFacts getMixedSetup() { + IndexFacts indexFacts = new IndexFacts(); + String musicDoctype = "music"; + String songDoctype = "song"; + Index musicDefault = new Index("default"); + musicDefault.setNGram(true, 1); + indexFacts.addIndex(musicDoctype, musicDefault); + Index songDefault = new Index("default"); + indexFacts.addIndex(songDoctype, songDefault); + Map<String, List<String>> clusters = new HashMap<>(); + clusters.put("musicOnly", Arrays.asList(new String[] { musicDoctype })); + clusters.put("songOnly", Arrays.asList(new String[] { songDoctype })); + clusters.put("musicAndSong", Arrays.asList(new String[] { musicDoctype, songDoctype })); + indexFacts.setClusters(clusters); + return indexFacts; + } + + public void testMixedDocTypes() { + final IndexFacts mixedSetup = getMixedSetup(); + { + Query q = new Query("?query=abc&restrict=song"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("abc", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&restrict=music"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=song"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("abc", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=music"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + } + + public void testMixedClusters() { + final IndexFacts mixedSetup = getMixedSetup(); + { + Query q = new Query("?query=abc&search=songOnly"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("abc", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=musicOnly"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=musicAndSong&restrict=music"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("AND a b c", q.getModel().getQueryTree().toString()); + } + { + Query q = new Query("?query=abc&search=musicAndSong&restrict=song"); + new Execution(searcher, Execution.Context.createContextStub(mixedSetup)).search(q); + assertEquals("abc", q.getModel().getQueryTree().toString()); + } + } + + public void testClusterMappingWithMixedDoctypes() { + final IndexFacts mixedSetup = getMixedSetup(); + + } + + public void testNGramRewritingMixedQuery() { + Query q=new Query("?query=foo+gram3:engul+test:bar"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND foo (AND gram3:eng gram3:ngu gram3:gul) test:bar",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingNGramOnly() { + Query q=new Query("?query=gram3:engul"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND gram3:eng gram3:ngu gram3:gul",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewriting2NGramsOnly() { + Query q=new Query("?query=gram3:engul+gram2:123"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND (AND gram3:eng gram3:ngu gram3:gul) (AND gram2:12 gram2:23)",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingShortOnly() { + Query q=new Query("?query=gram3:en"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("gram3:en",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingShortInMixes() { + Query q=new Query("?query=test:a+gram3:en"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND test:a gram3:en",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingPhrase() { + Query q=new Query("?query=gram3:%22engul+a+holi%22"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("gram3:\"eng ngu gul a hol oli\"",q.getModel().getQueryTree().toString()); + } + + /** + * Note that single-term phrases are simplified to just the term at parse time, + * so the ngram rewriter cannot know to keep the grams as a phrase in this case. + */ + public void testNGramRewritingPhraseSingleTerm() { + Query q=new Query("?query=gram3:%22engul%22"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND gram3:eng gram3:ngu gram3:gul",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingAdditionalTermInfo() { + Query q=new Query("?query=gram3:engul!50+foo+gram2:123!150"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + AndItem root=(AndItem)q.getModel().getQueryTree().getRoot(); + AndItem gram3And=(AndItem)root.getItem(0); + AndItem gram2And=(AndItem)root.getItem(2); + + assertExtraTermInfo(50,"engul",gram3And.getItem(0)); + assertExtraTermInfo(50,"engul",gram3And.getItem(1)); + assertExtraTermInfo(50,"engul",gram3And.getItem(2)); + assertExtraTermInfo(150,"123",gram2And.getItem(0)); + assertExtraTermInfo(150,"123",gram2And.getItem(1)); + } + + private void assertExtraTermInfo(int weight,String origin, Item g) { + WordItem gram=(WordItem)g; + assertEquals(weight,gram.getWeight()); + assertEquals(origin,gram.getOrigin().getValue()); + assertTrue(gram.isProtected()); + assertFalse(gram.isFromQuery()); + } + + public void testNGramRewritingExplicitDefault() { + Query q=new Query("?query=default:engul"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND default:eng default:ngu default:gul",q.getModel().getQueryTree().toString()); + } + + public void testNGramRewritingImplicitDefault() { + Query q=new Query("?query=engul"); + new Execution(searcher,Execution.Context.createContextStub(indexFacts)).search(q); + assertEquals("AND eng ngu gul",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentation() { + assertGramsWithSegmentation(new Chain<>(searcher)); + assertGramsWithSegmentation(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentation(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentation(Chain<Searcher> chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("gram14:firstsecondthird", Query.Type.ANY); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("AND gram14:firstsecondthi gram14:irstsecondthir gram14:rstsecondthird",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentationSingleSegment() { + assertGramsWithSegmentationSingleSegment(new Chain<>(searcher)); + assertGramsWithSegmentationSingleSegment(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentationSingleSegment(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentationSingleSegment(Chain<Searcher> chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("gram14:first", Query.Type.ANY); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("gram14:first",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentationSubstringSegmented() { + assertGramsWithSegmentationSubstringSegmented(new Chain<>(searcher)); + assertGramsWithSegmentationSubstringSegmented(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentationSubstringSegmented(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentationSubstringSegmented(Chain<Searcher> chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("gram14:afirstsecondthirdo", Query.Type.ANY); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("AND gram14:afirstsecondth gram14:firstsecondthi gram14:irstsecondthir gram14:rstsecondthird gram14:stsecondthirdo",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentationMixed() { + assertGramsWithSegmentationMixed(new Chain<>(searcher)); + assertGramsWithSegmentationMixed(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentationMixed(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentationMixed(Chain<Searcher> chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("a gram14:afirstsecondthird b gram14:hi", Query.Type.ALL); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("AND a (AND gram14:afirstsecondth gram14:firstsecondthi gram14:irstsecondthir gram14:rstsecondthird) b gram14:hi",q.getModel().getQueryTree().toString()); + } + + public void testGramsWithSegmentationMixedAndPhrases() { + assertGramsWithSegmentationMixedAndPhrases(new Chain<>(searcher)); + assertGramsWithSegmentationMixedAndPhrases(new Chain<>(new CJKSearcher(),searcher)); + assertGramsWithSegmentationMixedAndPhrases(new Chain<>(searcher,new CJKSearcher())); + } + public void assertGramsWithSegmentationMixedAndPhrases(Chain<Searcher> chain) { + // "first" "second" and "third" are segments in the "test" language + Item item = parseQuery("a gram14:\"afirstsecondthird b hi\"", Query.Type.ALL); + Query q=new Query("?query=ignored"); + q.getModel().setLanguage(Language.UNKNOWN); + q.getModel().getQueryTree().setRoot(item); + new Execution(chain,createContextStub(indexFacts)).search(q); + assertEquals("AND a gram14:\"afirstsecondth firstsecondthi irstsecondthir rstsecondthird b hi\"",q.getModel().getQueryTree().toString()); + } + + public void testNGramRecombining() { + Query q=new Query("?query=ignored"); + Result r=new Execution(new Chain<>(searcher,new MockBackend1()),createContextStub(indexFacts)).search(q); + Hit h1=r.hits().get("hit1"); + assertEquals("Should be untouched,\u001feven if containing \u001f",h1.getField("test").toString()); + assertTrue(h1.getField("test") instanceof String); + + assertEquals("Blue red Ed A",h1.getField("gram2").toString()); + assertTrue(h1.getField("gram2") instanceof XMLString); + + assertEquals("Separators on borders work","Blue red ed a\u001f",h1.getField("gram3").toString()); + assertTrue(h1.getField("gram3") instanceof String); + + Hit h2=r.hits().get("hit2"); + assertEquals("katt i...morgen",h2.getField("gram3").toString()); + assertTrue(h2.getField("gram3") instanceof JSONString); + + Hit h3=r.hits().get("hit3"); + assertEquals("\u001ffin\u001f \u001fen\u001f \u001fa\u001f",h3.getField("gram2").toString()); + assertEquals("#Logging in #Java is like that \"Judean P\u001fopul\u001far Front\" scene from \"Life of Brian\".", + h3.getField("gram3").toString()); + } + + private Item parseQuery(String query, Query.Type type) { + Parser parser = ParserFactory.newInstance(type, new ParserEnvironment().setIndexFacts(indexFacts)); + return parser.parse(new Parsable().setQuery(query).setLanguage(Language.UNKNOWN)).getRoot(); + } + + private static class MockBackend1 extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result r=new Result(query); + HitGroup g=new HitGroup(); + r.hits().add(g); + + Hit h1=new Hit("hit1"); + h1.setField(Hit.SDDOCNAME_FIELD,"default"); + h1.setField("test","Should be untouched,\u001feven if containing \u001f"); + h1.setField("gram2",new XMLString("\uFFF9Bl\uFFFAbl\uFFFBluue reed \uFFF9Ed\uFFFAed\uFFFB \uFFF9A\uFFFAa\uFFFB")); + h1.setField("gram3","\uFFF9Blu\uFFFAblu\uFFFBlue red ed a\u001f"); // separator on borders should not trip anything + g.add(h1); + + Hit h2=new Hit("hit2"); + h2.setField(Hit.SDDOCNAME_FIELD,"default"); + h2.setField("gram3",new JSONString("katatt i...mororgrgegen")); + r.hits().add(h2); + + // Test bolding + Hit h3=new Hit("hit3"); + h3.setField(Hit.SDDOCNAME_FIELD,"default"); + + // the result of searching for "fin en a" + h3.setField("gram2","\u001ffi\u001f\u001fin\u001f \u001fen\u001f \u001fa\u001f"); + + // the result from Juniper from of bolding the substring "opul": + h3.setField("gram3","#Logoggggigining in #Javava is likike thahat \"Jududedeaean Pop\u001fopu\u001f\u001fpul\u001fulalar Froronont\" scecenene frorom \"Lifife of Bririaian\"."); + r.hits().add(h3); + return r; + } + + } + + + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java new file mode 100644 index 00000000000..975eec9ce5c --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/QueryCombinatorTestCase.java @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform.test; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.querytransform.QueryCombinator; +import com.yahoo.search.searchchain.Execution; + +import junit.framework.TestCase; + +/** + * Unit testing of the searcher com.yahoo.search.querytransform.QueryCombinator. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class QueryCombinatorTestCase extends TestCase { + Searcher searcher; + + protected void setUp() throws Exception { + super.setUp(); + searcher = new QueryCombinator(new ComponentId("combinationTest")); + } + + protected void tearDown() throws Exception { + super.tearDown(); + } + + public void testStraightForwardSearch() { + Query q = new Query("?query=a&query.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a b", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a juhu.22[gnuff]:b", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&query.juhu="); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("a", q.getModel().getQueryTree().toString()); + q = new Query("?query=a+c&query.juhu=b"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("AND a c b", q.getModel().getQueryTree().toString()); + } + + public void testNoBaseQuery() { + Query q = new Query("?query.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("b", q.getModel().getQueryTree().toString()); + } + + public void testDefaultIndexWithoutQuery() { + Query q = new Query("?defidx.juhu=b"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("NULL", q.getModel().getQueryTree().toString()); + q = new Query("?query=a&defidx.juhu=b"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + assertEquals("a", q.getModel().getQueryTree().toString()); + } + + private static class StringPair { + public final String index; + public final String value; + + StringPair(String index, String value) { + super(); + this.index = index; + this.value = value; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((index == null) ? 0 : index.hashCode()); + result = prime * result + ((value == null) ? 0 : value.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + final StringPair other = (StringPair) obj; + if (index == null) { + if (other.index != null) + return false; + } else if (!index.equals(other.index)) + return false; + if (value == null) { + if (other.value != null) + return false; + } else if (!value.equals(other.value)) + return false; + return true; + } + + } + + public void testMultiPart() { + Query q = new Query("?query=a&query.juhu=b&query.nalle=c"); + Execution e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + Set<String> items = new HashSet<>(); + items.add("a"); + items.add("b"); + items.add("c"); + e.search(q); + // OK, the problem here is we have no way of knowing whether nalle or + // juhu was added first, since we have passed through HashMap instances + // inside the implementation + + AndItem root = (AndItem) q.getModel().getQueryTree().getRoot(); + Iterator<?> iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + if (items.contains(word.stringValue())) { + items.remove(word.stringValue()); + } else { + assertFalse("Got unexpected item in query tree: " + word.stringValue(), true); + } + } + assertEquals("Not all expected items found in query.", 0, items.size()); + + Set<StringPair> nastierItems = new HashSet<>(); + nastierItems.add(new StringPair("", "a")); + nastierItems.add(new StringPair("juhu.22[gnuff]", "b")); + nastierItems.add(new StringPair("gnuff[8].name(\"tralala\")", "c")); + q = new Query("?query=a&query.juhu=b&defidx.juhu=juhu.22[gnuff]&query.nalle=c&defidx.nalle=gnuff[8].name(%22tralala%22)"); + e = new Execution(searcher, Execution.Context.createContextStub(new IndexFacts())); + e.search(q); + root = (AndItem) q.getModel().getQueryTree().getRoot(); + iterator = root.getItemIterator(); + while (iterator.hasNext()) { + WordItem word = (WordItem) iterator.next(); + StringPair asPair = new StringPair(word.getIndexName(), word.stringValue()); + if (nastierItems.contains(asPair)) { + nastierItems.remove(asPair); + } else { + assertFalse("Got unexpected item in query tree: (" + + word.getIndexName() + ", " + word.stringValue() + ")", + true); + } + } + assertEquals("Not all expected items found in query.", 0, nastierItems.size()); + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/RangeQueryOptimizerTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/RangeQueryOptimizerTestCase.java new file mode 100644 index 00000000000..9362f729766 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/RangeQueryOptimizerTestCase.java @@ -0,0 +1,224 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.querytransform.RangeQueryOptimizer; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +import java.util.Iterator; + +import static org.junit.Assert.*; + +/** + * @author bratseth + */ +public class RangeQueryOptimizerTestCase { + + private static final Linguistics linguistics = new SimpleLinguistics(); + private static IndexFacts indexFacts = createIndexFacts(); + + @Test + public void testRangeOptimizing() { + assertOptimized("s:<15", "s:<15"); + assertOptimized("AND a s:[1999;2002]","a AND s:[1999;2002]"); + assertOptimized("AND s:<10;15>", "s:<15 AND s:>10"); + assertOptimized("AND s:give s:5 s:me", "s:give s:5 s:me"); + assertOptimized("AND s:[;15> b:<10;]", "s:<15 AND b:>10"); + assertOptimized("AND s:<10;15> b:[;20>", "s:<15 AND b:<20 AND s:>10"); + assertOptimized("AND c:foo s:<10;15> b:<35;40>", "s:<15 AND s:>10 b:>35 AND c:foo b:<40"); + assertOptimized("AND s:<12;15>", "s:<15 AND s:>10 AND s:>12"); + assertOptimized("Nonoverlapping ranges: Cannot match", "AND s:13 s:4 FALSE", "s:<15 AND s:>10 AND s:>100 AND s:13 AND s:<110 AND s:4"); + assertOptimized("Multivalue ranges are not optimized", "AND m:<15 m:>10", "m:<15 AND m:>10"); + assertOptimized("AND s:[13;15>", "s:<15 AND s:[13;17]"); + assertOptimized("AND s:[13;15>", "s:<15 AND s:[13;15]"); + assertOptimized("AND s:[13;15>", "s:[13;15] AND s:<15"); + assertOptimized("AND s:13 s:4 m:<100 s:[13;15> t:<101;109>", "s:<15 AND s:>10 AND t:>100 AND s:13 AND t:<110 AND s:4 AND t:>101 AND t:<111 AND t:<109 AND m:<100 AND s:[13;17]"); + assertOptimized("AND (AND s:<10;15>) (AND s:<22;27>)", "(s:<15 AND s:>10) AND (s:<27 AND s:>22 AND s:>20"); + assertOptimized("AND (AND s:<10;15.5>) (AND s:<22;27.37>)", "(s:<15.5 AND s:>10) AND (s:<27.37 AND s:>22 AND s:>20"); + assertOptimized("AND FALSE", "s:<2 AND s:>2"); + assertOptimized("AND FALSE", "s:>2 AND s:<2"); + assertOptimized("AND s:2", "s:[;2] AND s:[2;]"); + assertOptimized("AND s:2", "s:[2;] AND s:[;2]"); + } + + @Test + public void testRangeOptimizingCarriesOverItemAttributesWhenNotOptimized() { + Query query = new Query(); + AndItem root = new AndItem(); + query.getModel().getQueryTree().setRoot(root); + Item intItem = new IntItem(">" + 15, "s"); + intItem.setWeight(500); + intItem.setFilter(true); + intItem.setRanked(false); + root.addItem(intItem); + assertOptimized("Not optimized", "AND |s:<15;]!500", query); + IntItem transformedIntItem = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(0); + assertTrue("Filter was carried over", transformedIntItem.isFilter()); + assertFalse("Ranked was carried over", transformedIntItem.isRanked()); + assertEquals("Weight was carried over", 500, transformedIntItem.getWeight()); + } + + @Test + public void testRangeOptimizingCarriesOverItemAttributesWhenOptimized() { + Query query = new Query(); + AndItem root = new AndItem(); + query.getModel().getQueryTree().setRoot(root); + + Item intItem1 = new IntItem(">" + 15, "s"); + intItem1.setFilter(true); + intItem1.setRanked(false); + intItem1.setWeight(500); + root.addItem(intItem1); + + Item intItem2 = new IntItem("<" + 30, "s"); + intItem2.setFilter(true); + intItem2.setRanked(false); + intItem2.setWeight(500); + root.addItem(intItem2); + + assertOptimized("Optimized", "AND |s:<15;30>!500", query); + IntItem transformedIntItem = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(0); + assertTrue("Filter was carried over", transformedIntItem.isFilter()); + assertFalse("Ranked was carried over", transformedIntItem.isRanked()); + assertEquals("Weight was carried over", 500, transformedIntItem.getWeight()); + } + + @Test + public void testNoRangeOptimizingWhenAttributesAreIncompatible() { + Query query = new Query(); + AndItem root = new AndItem(); + query.getModel().getQueryTree().setRoot(root); + + Item intItem1 = new IntItem(">" + 15, "s"); + intItem1.setFilter(true); + intItem1.setRanked(false); + intItem1.setWeight(500); + root.addItem(intItem1); + + Item intItem2 = new IntItem("<" + 30, "s"); + intItem2.setFilter(false); // Disagrees with item1 + intItem2.setRanked(false); + intItem2.setWeight(500); + root.addItem(intItem2); + + assertOptimized("Not optimized", "AND |s:<15;]!500 s:[;30>!500", query); + + IntItem transformedIntItem1 = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(0); + assertTrue("Filter was carried over", transformedIntItem1.isFilter()); + assertFalse("Ranked was carried over", transformedIntItem1.isRanked()); + assertEquals("Weight was carried over", 500, transformedIntItem1.getWeight()); + + IntItem transformedIntItem2 = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(1); + assertFalse("Filter was carried over", transformedIntItem2.isFilter()); + assertFalse("Ranked was carried over", transformedIntItem2.isRanked()); + assertEquals("Weight was carried over", 500, transformedIntItem2.getWeight()); + } + + @Test + public void testDifferentCompatibleRangesPerFieldAreOptimizedSeparately() { + Query query = new Query(); + AndItem root = new AndItem(); + query.getModel().getQueryTree().setRoot(root); + + // Two internally compatible items + Item intItem1 = new IntItem(">" + 15, "s"); + intItem1.setRanked(false); + root.addItem(intItem1); + + Item intItem2 = new IntItem("<" + 30, "s"); + intItem2.setRanked(false); + root.addItem(intItem2); + + // Two other internally compatible items incompatible with the above + Item intItem3 = new IntItem(">" + 100, "s"); + root.addItem(intItem3); + + Item intItem4 = new IntItem("<" + 150, "s"); + root.addItem(intItem4); + + assertOptimized("Optimized", "AND s:<15;30> s:<100;150>", query); + + IntItem transformedIntItem1 = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(0); + assertFalse("Ranked was carried over", transformedIntItem1.isRanked()); + + IntItem transformedIntItem2 = (IntItem)((AndItem)query.getModel().getQueryTree().getRoot()).getItem(1); + assertTrue("Ranked was carried over", transformedIntItem2.isRanked()); + } + + @Test + public void assertOptmimizedYQLQuery() { + Query query = new Query("/?query=select%20%2A%20from%20sources%20%2A%20where%20%28range%28s%2C%20100000%2C%20100000%29%20OR%20range%28t%2C%20-20000000000L%2C%20-20000000000L%29%20OR%20range%28t%2C%2030%2C%2030%29%29%3B&type=yql"); + assertOptimized("YQL usage of the IntItem API works", "OR s:100000 t:-20000000000 t:30", query); + } + + @Test + public void testTracing() { + Query notOptimized = new Query("/?tracelevel=2"); + notOptimized.getModel().getQueryTree().setRoot(parseQuery("s:<15")); + assertOptimized("", "s:<15", notOptimized); + assertFalse(contains("Optimized query ranges", notOptimized.getContext(true).getTrace().traceNode().descendants(String.class))); + + Query optimized = new Query("/?tracelevel=2"); + optimized.getModel().getQueryTree().setRoot(parseQuery("s:<15 AND s:>10")); + assertOptimized("", "AND s:<10;15>", optimized); + assertTrue(contains("Optimized query ranges", optimized.getContext(true).getTrace().traceNode().descendants(String.class))); + } + + private boolean contains(String prefix, Iterable<String> traceEntries) { + for (String traceEntry : traceEntries) + if (traceEntry.startsWith(prefix)) return true; + return false; + } + + private Query assertOptimized(String expected, String queryString) { + return assertOptimized(null, expected, queryString); + } + + private Query assertOptimized(String explanation, String expected, String queryString) { + Query query = new Query(); + query.getModel().getQueryTree().setRoot(parseQuery(queryString)); + return assertOptimized(explanation, expected, query); + } + + private Query assertOptimized(String explanation, String expected, Query query) { + Chain<Searcher> chain = new Chain<>("test", new RangeQueryOptimizer()); + new Execution(chain, Execution.Context.createContextStub(indexFacts)).search(query); + assertEquals(explanation, expected, query.getModel().getQueryTree().getRoot().toString()); + return query; + } + + private Item parseQuery(String query) { + IndexFacts indexFacts = new IndexFacts(); + Parser parser = ParserFactory.newInstance(Query.Type.ADVANCED, new ParserEnvironment() + .setIndexFacts(indexFacts) + .setLinguistics(linguistics)); + return parser.parse(new Parsable().setQuery(query)).getRoot(); + } + + private static IndexFacts createIndexFacts() { + IndexFacts indexFacts = new IndexFacts(); + Index singleValue1 = new Index("s"); + Index singleValue2 = new Index("t"); + Index multiValue = new Index("m"); + multiValue.setMultivalue(true); + indexFacts.addIndex("test", singleValue1); + indexFacts.addIndex("test", singleValue2); + indexFacts.addIndex("test", multiValue); + return indexFacts; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java new file mode 100644 index 00000000000..8e645f2781b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/SortingDegraderTestCase.java @@ -0,0 +1,173 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.querytransform.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.GroupingQueryParser; +import com.yahoo.search.query.properties.DefaultProperties; +import com.yahoo.search.querytransform.SortingDegrader; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class SortingDegraderTestCase { + + @Test + public void testDegradingAscending() { + Query query = new Query("?ranking.sorting=%2ba1%20-a2"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertTrue(query.getRanking().getMatchPhase().getAscending()); + assertEquals(1400l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + assertEquals(1.0, query.getRanking().getMatchPhase().getMaxFilterCoverage().doubleValue(), 1e-16); + } + + @Test + public void testDegradingDescending() { + Query query = new Query("?ranking.sorting=-a1%20-a2"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertFalse(query.getRanking().getMatchPhase().getAscending()); + assertEquals(1400l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + } + + @Test + public void testDegradingNonDefaultMaxHits() { + Query query = new Query("?ranking.sorting=-a1%20-a2&ranking.matchPhase.maxHits=37"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertFalse(query.getRanking().getMatchPhase().getAscending()); + assertEquals(37l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + } + + @Test + public void testDegradingNonDefaultMaxFilterCoverage() { + Query query = new Query("?ranking.sorting=-a1%20-a2&ranking.matchPhase.maxFilterCoverage=0.37"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertFalse(query.getRanking().getMatchPhase().getAscending()); + assertEquals(0.37d, query.getRanking().getMatchPhase().getMaxFilterCoverage().doubleValue(), 1e-16); + } + + @Test + public void testDegradingNonDefaultIllegalMaxFilterCoverage() { + try { + Query query = new Query("?ranking.sorting=-a1%20-a2&ranking.matchPhase.maxFilterCoverage=37"); + assertTrue(false); + } catch (QueryException qe) { + assertEquals("Invalid request parameter", qe.getMessage()); + Throwable setE = qe.getCause(); + assertTrue(setE instanceof IllegalArgumentException); + assertEquals("Could not set 'ranking.matchPhase.maxFilterCoverage' to '37'", setE.getMessage()); + Throwable rootE = setE.getCause(); + assertTrue(rootE instanceof IllegalArgumentException); + assertEquals("maxFilterCoverage must be in the range [0.0, 1.0]. It is 37.0", rootE.getMessage()); + } + + } + + @Test + public void testNoDegradingWhenGrouping() { + Query query = new Query("?ranking.sorting=%2ba1%20-a2&select=all(group(a1)%20each(output(a1)))"); + execute(query); + assertNull(query.getRanking().getMatchPhase().getAttribute()); + } + + @Test + public void testNoDegradingWhenNonFastSearchAttribute() { + Query query = new Query("?ranking.sorting=%2bnonFastSearchAttribute%20-a2"); + execute(query); + assertNull(query.getRanking().getMatchPhase().getAttribute()); + } + + @Test + public void testNoDegradingWhenNonNumericalAttribute() { + Query query = new Query("?ranking.sorting=%2bstringAttribute%20-a2"); + execute(query); + assertNull(query.getRanking().getMatchPhase().getAttribute()); + } + + @Test + public void testNoDegradingWhenTurnedOff() { + Query query = new Query("?ranking.sorting=-a1%20-a2&sorting.degrading=false"); + execute(query); + assertNull(query.getRanking().getMatchPhase().getAttribute()); + } + + @Test + public void testAccessAllDegradingParametersInQuery() { + Query query = new Query("?ranking.matchPhase.maxHits=555&ranking.matchPhase.attribute=foo&ranking.matchPhase.ascending=true"); + execute(query); + + assertEquals("foo", query.getRanking().getMatchPhase().getAttribute()); + assertTrue(query.getRanking().getMatchPhase().getAscending()); + assertEquals(555l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + + assertEquals("foo", query.properties().get("ranking.matchPhase.attribute")); + assertTrue(query.properties().getBoolean("ranking.matchPhase.ascending")); + assertEquals(555l, query.properties().getLong("ranking.matchPhase.maxHits").longValue()); + } + + @Test + public void testDegradingWithLargeMaxHits() { + Query query = new Query("?ranking.sorting=%2ba1%20-a2"); + query.properties().set(DefaultProperties.MAX_HITS, 13 * 1000); + query.properties().set(DefaultProperties.MAX_OFFSET, 8 * 1000); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertTrue(query.getRanking().getMatchPhase().getAscending()); + assertEquals(21000l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + } + + @Test + public void testDegradingWithoutPaginationSupport() { + Query query = new Query("?ranking.sorting=%2ba1%20-a2&hits=7&offset=1"); + query.properties().set(DefaultProperties.MAX_HITS, 13 * 1000); + query.properties().set(DefaultProperties.MAX_OFFSET, 8 * 1000); + query.properties().set(SortingDegrader.PAGINATION, "false"); + execute(query); + assertEquals("a1", query.getRanking().getMatchPhase().getAttribute()); + assertTrue(query.getRanking().getMatchPhase().getAscending()); + assertEquals(8l, query.getRanking().getMatchPhase().getMaxHits().longValue()); + } + + private Result execute(Query query) { + // Add the grouping parser to transfer the select parameter to a grouping expression + Chain<Searcher> chain = new Chain<Searcher>(new GroupingQueryParser(), new SortingDegrader()); + return new Execution(chain, Execution.Context.createContextStub(createIndexFacts())).search(query); + } + + private IndexFacts createIndexFacts() { + IndexFacts indexFacts = new IndexFacts(); + + Index fastSearchAttribute1 = new Index("a1"); + fastSearchAttribute1.setFastSearch(true); + fastSearchAttribute1.setNumerical(true); + + Index fastSearchAttribute2 = new Index("a2"); + fastSearchAttribute2.setFastSearch(true); + fastSearchAttribute2.setNumerical(true); + + Index nonFastSearchAttribute = new Index("nonFastSearchAttribute"); + nonFastSearchAttribute.setNumerical(true); + + Index stringAttribute = new Index("stringAttribute"); + stringAttribute.setFastSearch(true); + + indexFacts.addIndex("test", fastSearchAttribute1); + indexFacts.addIndex("test", fastSearchAttribute2); + indexFacts.addIndex("test", nonFastSearchAttribute); + indexFacts.addIndex("stringAttribute", stringAttribute); + return indexFacts; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java new file mode 100644 index 00000000000..fa690138e88 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java @@ -0,0 +1,144 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.rendering; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.junit.Test; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.concurrent.Receiver; +import com.yahoo.processing.response.Data; +import com.yahoo.processing.response.DataList; +import com.yahoo.processing.response.DefaultIncomingData; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.text.Utf8; + +/** + * Test adding hits to a hit group during rendering. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class AsyncGroupPopulationTestCase { + private static class WrappedFuture<F> implements ListenableFuture<F> { + Receiver<Boolean> isListening = new Receiver<>(); + + private ListenableFuture<F> wrapped; + + WrappedFuture(ListenableFuture<F> wrapped) { + this.wrapped = wrapped; + } + + public void addListener(Runnable listener, Executor executor) { + wrapped.addListener(listener, executor); + isListening.put(Boolean.TRUE); + } + + public boolean cancel(boolean mayInterruptIfRunning) { + return wrapped.cancel(mayInterruptIfRunning); + } + + public boolean isCancelled() { + return wrapped.isCancelled(); + } + + public boolean isDone() { + return wrapped.isDone(); + } + + public F get() throws InterruptedException, ExecutionException { + return wrapped.get(); + } + + public F get(long timeout, TimeUnit unit) throws InterruptedException, + ExecutionException, TimeoutException { + return wrapped.get(timeout, unit); + } + } + + private static class ObservableIncoming<DATATYPE extends Data> extends DefaultIncomingData<DATATYPE> { + WrappedFuture<DataList<DATATYPE>> waitForIt = null; + private final Object lock = new Object(); + + @Override + public ListenableFuture<DataList<DATATYPE>> completed() { + synchronized (lock) { + if (waitForIt == null) { + waitForIt = new WrappedFuture<>(super.completed()); + } + } + return waitForIt; + } + } + + private static class InstrumentedGroup extends HitGroup { + private static final long serialVersionUID = 4585896586414935558L; + + InstrumentedGroup(String id) { + super(id, new Relevance(1), new ObservableIncoming<Hit>()); + ((ObservableIncoming<Hit>) incoming()).assignOwner(this); + } + + } + + @Test + public final void test() throws InterruptedException, ExecutionException, + JsonParseException, JsonMappingException, IOException { + String rawExpected = "{" + + " \"root\": {" + + " \"children\": [" + + " {" + + " \"id\": \"yahoo1\"," + + " \"relevance\": 1.0" + + " }," + + " {" + + " \"id\": \"yahoo2\"," + + " \"relevance\": 1.0" + + " }" + + " ]," + + " \"fields\": {" + + " \"totalCount\": 0" + + " }," + + " \"id\": \"yahoo\"," + + " \"relevance\": 1.0" + + " }" + + "}"; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HitGroup h = new InstrumentedGroup("yahoo"); + h.incoming().add(new Hit("yahoo1")); + JsonRenderer renderer = new JsonRenderer(); + Result result = new Result(new Query(), h); + renderer.init(); + ListenableFuture<Boolean> f = renderer.render(out, result, + new Execution(Execution.Context.createContextStub()), + result.getQuery()); + WrappedFuture<DataList<Hit>> x = (WrappedFuture<DataList<Hit>>) h.incoming().completed(); + x.isListening.get(86_400_000); + h.incoming().add(new Hit("yahoo2")); + h.incoming().markComplete(); + Boolean b = f.get(); + assertTrue(b); + String rawGot = Utf8.toString(out.toByteArray()); + ObjectMapper m = new ObjectMapper(); + Map<?, ?> expected = m.readValue(rawExpected, Map.class); + Map<?, ?> got = m.readValue(rawGot, Map.class); + assertEquals(expected, got); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java new file mode 100644 index 00000000000..4b26187c9d3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -0,0 +1,1111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.rendering; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.times; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.document.predicate.Predicate; + +import com.yahoo.tensor.MapTensor; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.fasterxml.jackson.core.JsonGenerationException; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.chain.Chain; +import com.yahoo.data.access.slime.SlimeAdapter; +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentId; +import com.yahoo.document.Field; +import com.yahoo.document.StructDataType; +import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.document.datatypes.Struct; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.hitfield.JSONString; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.grouping.result.DoubleBucketId; +import com.yahoo.search.grouping.result.Group; +import com.yahoo.search.grouping.result.GroupList; +import com.yahoo.search.grouping.result.RootGroup; +import com.yahoo.search.grouping.result.StringId; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.result.NanNumber; +import com.yahoo.search.result.Relevance; +import com.yahoo.search.result.StructuredData; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.statistics.ElapsedTimeTestCase; +import com.yahoo.search.statistics.TimeTracker; +import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; +import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.text.Utf8; +import com.yahoo.yolean.trace.TraceNode; + +import org.mockito.Mockito; + +/** + * Functional testing of {@link JsonRenderer}. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class JsonRendererTestCase { + + JsonRenderer originalRenderer; + JsonRenderer renderer; + + public JsonRendererTestCase() { + originalRenderer = new JsonRenderer(); + } + + @Before + public void setUp() throws Exception { + // Do the same dance as in production + renderer = (JsonRenderer) originalRenderer.clone(); + renderer.init(); + } + + @After + public void tearDown() throws Exception { + renderer = null; + } + + private static final class Thingie { + @Override + public String toString() { + return "thingie"; + } + } + + @Test + public final void testDocumentId() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"documentid\": \"id:unittest:smoke::whee\"\n" + + " },\n" + + " \"id\": \"id:unittest:smoke::whee\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("docIdTest"); + h.setField("documentid", new DocumentId("id:unittest:smoke::whee")); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + private Result newEmptyResult() { + Query q = new Query("/?query=a"); + Result r = new Result(q); + return r; + } + + @Test + public final void testDataTypes() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"double\": 0.00390625,\n" + + " \"float\": 14.29,\n" + + " \"integer\": 1,\n" + + " \"long\": 4398046511104,\n" + + " \"object\": \"thingie\",\n" + + " \"string\": \"stuff\",\n" + + " \"predicate\": \"a in [b]\",\n" + + " \"tensor\": { \"dimensions\": [\"x\"], \n" + + " \"cells\": [ { \"address\": {\"x\": \"a\"}, \"value\":2.0 } ] }\n" + + " },\n" + + " \"id\": \"datatypestuff\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("datatypestuff"); + // the floating point values are chosen to get a deterministic string representation + h.setField("double", Double.valueOf(0.00390625d)); + h.setField("float", Float.valueOf(14.29f)); + h.setField("integer", Integer.valueOf(1)); + h.setField("long", Long.valueOf(4398046511104L)); + h.setField("string", "stuff"); + h.setField("predicate", Predicate.fromString("a in [b]")); + h.setField("tensor", new TensorFieldValue(MapTensor.from("{ {x:a}: 2.0}"))); + h.setField("object", new Thingie()); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + + @Test + public final void testTracing() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + // which clearly shows a trace child is created once too often... + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"something\"\n" + + " },\n" + + " {\n" + + " \"message\": \"something else\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"yellow\"\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"message\": \"marker\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=1"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + + execution.search(q); + q.trace("something", 1); + q.trace("something else", 1); + Execution e2 = new Execution(new Chain<Searcher>(), execution.context()); + Query subQuery = new Query("/?query=b&tracelevel=1"); + e2.search(subQuery); + subQuery.trace("yellow", 1); + q.trace("marker", 1); + String summary = render(execution, r); + assertEqualJson(expected, summary); + } + + @Test + public final void testEmptyTracing() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=0"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + + execution.search(q); + Execution e2 = new Execution(new Chain<Searcher>(), execution.context()); + Query subQuery = new Query("/?query=b&tracelevel=0"); + e2.search(subQuery); + subQuery.trace("yellow", 1); + q.trace("marker", 1); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture<Boolean> f = renderer.render(bs, r, execution, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + assertEqualJson(expected, summary); + } + + @SuppressWarnings("unchecked") + @Test + public final void testTracingWithEmptySubtree() throws IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"message\": \"Resolved properties:\\ntracelevel=10 (value from request)\\nquery=a (value from request)\\n\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"timestamp\": 42\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + Query q = new Query("/?query=a&tracelevel=10"); + Execution execution = new Execution(Execution.Context.createContextStub()); + Result r = new Result(q); + + execution.search(q); + new Execution(new Chain<Searcher>(), execution.context()); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture<Boolean> f = renderer.render(bs, r, execution, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + ObjectMapper m = new ObjectMapper(); + + Map<String, Object> exp = m.readValue(expected, Map.class); + Map<String, Object> gen = m.readValue(summary, Map.class); + { + // nuke timestamp and check it's there + Map<String, Object> trace = (Map<String, Object>) gen.get("trace"); + List<Object> children1 = (List<Object>) trace.get("children"); + Map<String, Object> subtrace = (Map<String, Object>) children1.get(2); + List<Object> children2 = (List<Object>) subtrace.get("children"); + Map<String, Object> traceElement = (Map<String, Object>) children2.get(0); + traceElement.put("timestamp", Integer.valueOf(42)); + } + assertEquals(exp, gen); + } + + + @Test + public final void testHalfEmptyTracing() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"green\"" + + " }" + + " ]" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=0"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + + execution.search(q); + subExecution(execution, "red", 0); + subExecution(execution, "green", 1); + subExecution(execution, "blue", 0); + q.trace("marker", 1); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture<Boolean> f = renderer.render(bs, r, execution, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + assertEqualJson(expected, summary); + } + + private void subExecution(Execution execution, String color, int traceLevel) { + Execution e2 = new Execution(new Chain<Searcher>(), execution.context()); + Query subQuery = new Query("/?query=b&tracelevel=" + traceLevel); + e2.search(subQuery); + subQuery.trace(color, 1); + } + + @Test + public final void testTracingOfNodesWithBothChildrenAndData() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"string payload\",\n" + + " \"children\": [" + + " {\n" + + " \"message\": \"leafnode\"" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"message\": \"something\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=1"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + execution.search(q); + final TraceNode child = new TraceNode("string payload", 0L); + child.add(new TraceNode("leafnode", 0L)); + execution.trace().traceNode().add(child); + q.trace("something", 1); + String summary = render(execution, r); + assertEqualJson(expected, summary); + } + + + @Test + public final void testTracingOfNodesWithBothChildrenAndDataAndEmptySubnode() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"string payload\"\n" + + " },\n" + + " {\n" + + " \"message\": \"something\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=1"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + execution.search(q); + final TraceNode child = new TraceNode("string payload", 0L); + child.add(new TraceNode(null, 0L)); + execution.trace().traceNode().add(child); + q.trace("something", 1); + String summary = render(execution, r); + assertEqualJson(expected, summary); + } + + @Test + public final void testTracingOfNestedNodesWithDataAndSubnodes() throws JsonGenerationException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " \"trace\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"No query profile is used\"\n" + + " },\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"string payload\",\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"message\": \"in OO languages, nesting is for birds\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=1"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + execution.search(q); + final TraceNode child = new TraceNode("string payload", 0L); + final TraceNode childOfChild = new TraceNode(null, 0L); + child.add(childOfChild); + childOfChild.add(new TraceNode("in OO languages, nesting is for birds", 0L)); + execution.trace().traceNode().add(child); + String summary = render(execution, r); + assertEqualJson(expected, summary); + } + + + @Test + public final void test() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"c\": \"d\",\n" + + " \"uri\": \"http://localhost/1\"\n" + + " },\n" + + " \"id\": \"http://localhost/1\",\n" + + " \"relevance\": 0.9,\n" + + " \"types\": [\n" + + " \"summary\"\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"id\": \"usual\",\n" + + " \"relevance\": 1.0\n" + + " },\n" + + " {\n" + + " \"fields\": {\n" + + " \"e\": \"f\"\n" + + " },\n" + + " \"id\": \"type grouphit\",\n" + + " \"relevance\": 1.0,\n" + + " \"types\": [\n" + + " \"grouphit\"\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"fields\": {\n" + + " \"b\": \"foo\",\n" + + " \"uri\": \"http://localhost/\"\n" + + " },\n" + + " \"id\": \"http://localhost/\",\n" + + " \"relevance\": 0.95,\n" + + " \"types\": [\n" + + " \"summary\"\n" + + " ]\n" + + " }\n" + + " ],\n" + + " \"coverage\": {\n" + + " \"coverage\": 100,\n" + + " \"documents\": 500,\n" + + " \"full\": true,\n" + + " \"nodes\": 1,\n" + + " \"results\": 1,\n" + + " \"resultsFull\": 1\n" + + " },\n" + + " \"errors\": [\n" + + " {\n" + + " \"code\": 18,\n" + + " \"message\": \"boom\",\n" + + " \"summary\": \"Internal server error.\"\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}"; + Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Execution execution = new Execution( + Execution.Context.createContextStub()); + Result r = new Result(q); + r.setCoverage(new Coverage(500, 1, true)); + + FastHit h = new FastHit("http://localhost/", .95); + h.setField("$a", "Hello, world."); + h.setField("b", "foo"); + r.hits().add(h); + HitGroup g = new HitGroup("usual"); + h = new FastHit("http://localhost/1", .90); + h.setField("c", "d"); + g.add(h); + r.hits().add(g); + HitGroup gg = new HitGroup("type grouphit"); + gg.types().add("grouphit"); + gg.setField("e", "f"); + r.hits().add(gg); + r.hits().addError(ErrorMessage.createInternalServerError("boom")); + String summary = render(execution, r); + // System.out.println(summary); + assertEqualJson(expected, summary); + } + + @Test + public void testMoreTypes() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"bigDecimal\": 3.402823669209385e+38,\n" + + " \"bigInteger\": 340282366920938463463374607431768211455,\n" + + " \"byte\": 8,\n" + + " \"short\": 16\n" + + " },\n" + + " \"id\": \"moredatatypestuff\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("moredatatypestuff"); + h.setField("byte", Byte.valueOf((byte) 8)); + h.setField("short", Short.valueOf((short) 16)); + h.setField("bigInteger", new BigInteger( + "340282366920938463463374607431768211455")); + h.setField("bigDecimal", new BigDecimal( + "340282366920938463463374607431768211456.5")); + h.setField("nanNumber", NanNumber.NaN); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testNullField() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"null\": null\n" + + " },\n" + + " \"id\": \"nullstuff\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("nullstuff"); + h.setField("null", null); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testLazyDecoding() throws IOException { + FastHit f = new FastHit("http://a.b/c", 0.5); + String checkWeCanDecode = "bamse"; + String dontCare = "don't care"; + final String fieldName = "checkWeCanDecode"; + f.setLazyStringField(fieldName, Utf8.toBytes(checkWeCanDecode)); + final String fieldName2 = "dontCare"; + f.setLazyStringField(fieldName2, Utf8.toBytes(dontCare)); + assertEquals(checkWeCanDecode, f.getField(fieldName)); + + JsonGenerator mock = Mockito.mock(JsonGenerator.class); + + renderer.setGenerator(mock); + assertTrue(renderer.tryDirectRendering(fieldName2, f)); + + byte[] expectedBytes = Utf8.toBytes(dontCare); + Mockito.verify(mock, times(1)).writeUTF8String(expectedBytes, 0, expectedBytes.length); + } + + @Test + public void testHitWithSource() throws JsonParseException, JsonMappingException, IOException, InterruptedException, ExecutionException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"id\": \"datatypestuff\",\n" + + " \"relevance\": 1.0,\n" + + " \"source\": \"unit test\"\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("datatypestuff"); + h.setSource("unit test"); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testErrorWithStackTrace() throws InterruptedException, + ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"errors\": [\n" + + " {\n" + + " \"code\": 1234,\n" + + " \"message\": \"top of the day\",\n" + + " \"stackTrace\": \"java.lang.Throwable\\n\\tat com.yahoo.search.rendering.JsonRendererTestCase.testErrorWithStackTrace(JsonRendererTestCase.java:732)\\n\",\n" + + " \"summary\": \"hello\"\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + Result r = new Result(q); + Throwable t = new Throwable(); + StackTraceElement[] stack = new StackTraceElement[1]; + stack[0] = new StackTraceElement( + "com.yahoo.search.rendering.JsonRendererTestCase", + "testErrorWithStackTrace", "JsonRendererTestCase.java", 732); + t.setStackTrace(stack); + ErrorMessage e = new ErrorMessage(1234, "hello", "top of the day", t); + r.hits().addError(e); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testContentHeader() { + assertEquals("utf-8", renderer.getEncoding()); + assertEquals("application/json", renderer.getMimeType()); + } + + @Test + public void testGrouping() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"count()\": 7\n" + + " },\n" + + " \"value\": \"Jones\",\n" + + " \"id\": \"group:string:Jones\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"continuation\": {\n" + + " \"next\": \"CCCC\",\n" + + " \"prev\": \"BBBB\"\n" + + " },\n" + + " \"id\": \"grouplist:customer\",\n" + + " \"label\": \"customer\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"continuation\": {\n" + + " \"this\": \"AAAA\"\n" + + " },\n" + + " \"id\": \"group:root:0\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + RootGroup rg = new RootGroup(0, new Continuation() { + @Override + public String toString() { + return "AAAA"; + } + }); + GroupList gl = new GroupList("customer"); + gl.continuations().put("prev", new Continuation() { + @Override + public String toString() { + return "BBBB"; + } + }); + gl.continuations().put("next", new Continuation() { + @Override + public String toString() { + return "CCCC"; + } + }); + Group g = new Group(new StringId("Jones"), new Relevance(1.0)); + g.setField("count()", Integer.valueOf(7)); + gl.add(g); + rg.add(gl); + r.hits().add(rg); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testGroupingWithBucket() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"something()\": 7\n" + + " },\n" + + " \"limits\": {\n" + + " \"from\": \"1.0\",\n" + + " \"to\": \"2.0\"\n" + + " },\n" + + " \"id\": \"group:double_bucket:1.0:2.0\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"id\": \"grouplist:customer\",\n" + + " \"label\": \"customer\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"continuation\": {\n" + + " \"this\": \"AAAA\"\n" + + " },\n" + + " \"id\": \"group:root:0\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + RootGroup rg = new RootGroup(0, new Continuation() { + @Override + public String toString() { + return "AAAA"; + } + }); + GroupList gl = new GroupList("customer"); + Group g = new Group(new DoubleBucketId(1.0, 2.0), new Relevance(1.0)); + g.setField("something()", Integer.valueOf(7)); + gl.add(g); + rg.add(gl); + r.hits().add(rg); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public void testJsonObjects() throws JsonParseException, JsonMappingException, InterruptedException, ExecutionException, IOException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"inspectable\": {\n" + + " \"a\": \"b\"\n" + + " },\n" + + " \"jackson\": {\n" + + " \"Nineteen-eighty-four\": 1984\n" + + " },\n" + + " \"json producer\": {\n" + + " \"long in structured\": 7809531904\n" + + " },\n" + + " \"org.json array\": [\n" + + " true,\n" + + " true,\n" + + " false\n" + + " ],\n" + + " \"org.json object\": {\n" + + " \"forty-two\": 42\n" + + " }\n" + + " },\n" + + " \"id\": \"json objects\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 0\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("json objects"); + JSONObject o = new JSONObject(); + JSONArray a = new JSONArray(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode j = mapper.createObjectNode(); + JSONString s = new JSONString("{\"a\": \"b\"}"); + Slime slime = new Slime(); + Cursor c = slime.setObject(); + c.setLong("long in structured", 7809531904L); + SlimeAdapter slimeInit = new SlimeAdapter(slime.get()); + StructuredData struct = new StructuredData(slimeInit); + ((ObjectNode) j).put("Nineteen-eighty-four", 1984); + o.put("forty-two", 42); + a.put(true); + a.put(true); + a.put(false); + h.setField("inspectable", s); + h.setField("jackson", j); + h.setField("json producer", struct); + h.setField("org.json array", a); + h.setField("org.json object", o); + r.hits().add(h); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public final void testFieldValueInHit() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"fromDocumentApi\":{\"integerField\":123, \"stringField\":\"abc\"}" + + " },\n" + + " \"id\": \"fieldValueTest\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = new Hit("fieldValueTest"); + StructDataType structType = new StructDataType("jsonRenderer"); + structType.addField(new Field("stringField", DataType.STRING)); + structType.addField(new Field("integerField", DataType.INT)); + Struct struct = structType.createFieldValue(); + struct.setFieldValue("stringField", "abc"); + struct.setFieldValue("integerField", 123); + h.setField("fromDocumentApi", struct); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public final void testHiddenFields() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"id\": \"hiddenFields\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = newEmptyResult(); + Hit h = createHitWithOnlyHiddenFields(); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + private Hit createHitWithOnlyHiddenFields() { + Hit h = new Hit("hiddenFields"); + h.setField("NaN", NanNumber.NaN); + h.setField("emptyString", ""); + h.setField("emptyStringFieldValue", new StringFieldValue("")); + h.setField("$vespaImplementationDetail", "Hello, World!"); + return h; + } + + @Test + public final void testDebugRendering() throws IOException, InterruptedException, ExecutionException, JSONException { + String expected = "{\n" + + " \"root\": {\n" + + " \"children\": [\n" + + " {\n" + + " \"fields\": {\n" + + " \"NaN\": \"NaN\",\n" + + " \"emptyString\": \"\",\n" + + " \"emptyStringFieldValue\": \"\",\n" + + " \"$vespaImplementationDetail\": \"Hello, World!\"\n" + + " },\n" + + " \"id\": \"hiddenFields\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + " ],\n" + + " \"fields\": {\n" + + " \"totalCount\": 1\n" + + " },\n" + + " \"id\": \"toplevel\",\n" + + " \"relevance\": 1.0\n" + + " }\n" + + "}\n"; + Result r = new Result(new Query("/?renderer.json.debug=true")); + Hit h = createHitWithOnlyHiddenFields(); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test + public final void testTimingRendering() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException { + String expected = "{" + + " \"root\": {" + + " \"fields\": {" + + " \"totalCount\": 0" + + " }," + + " \"id\": \"toplevel\"," + + " \"relevance\": 1.0" + + " }," + + " \"timing\": {" + + " \"querytime\": 0.006," + + " \"searchtime\": 0.007," + + " \"summaryfetchtime\": 0.0" + + " }" + + "}"; + Result r = new Result(new Query("/?renderer.json.debug=true&presentation.timing=true")); + TimeTracker t = new TimeTracker(new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource( + new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + r.getElapsedTime().add(t); + renderer.setTimeSource(() -> 8L); + String summary = render(r); + System.out.println(summary); + assertEqualJson(expected, summary); + } + + private String render(Result r) throws InterruptedException, + ExecutionException { + Execution execution = new Execution( + Execution.Context.createContextStub()); + return render(execution, r); + } + + private String render(Execution execution, Result r) + throws InterruptedException, ExecutionException { + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture<Boolean> f = renderer.render(bs, r, execution, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + return summary; + } + + @SuppressWarnings("unchecked") + private void assertEqualJson(String expected, String generated) throws JsonParseException, JsonMappingException, IOException { + ObjectMapper m = new ObjectMapper(); + Map<String, Object> exp = m.readValue(expected, Map.class); + Map<String, Object> gen = m.readValue(generated, Map.class); + assertEquals(exp, gen); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java new file mode 100644 index 00000000000..dc0bc42d410 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/rendering/SyncDefaultRendererTestCase.java @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.rendering; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.statistics.ElapsedTimeTestCase; +import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; +import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; +import com.yahoo.search.statistics.TimeTracker; +import com.yahoo.text.Utf8; + +/** + * Check the legacy sync default renderer doesn't spontaneously combust. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class SyncDefaultRendererTestCase { + + SyncDefaultRenderer d; + + @Before + public void setUp() throws Exception { + d = new SyncDefaultRenderer(); + d.init(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetEncoding() { + assertEquals("utf-8", d.getEncoding()); + } + + @Test + public final void testGetMimeType() { + assertEquals("text/xml", d.getMimeType()); + } + + @SuppressWarnings("deprecation") + @Test + public final void testRenderWriterResult() throws IOException, InterruptedException, ExecutionException { + Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + q.getPresentation().setTiming(true); + Result r = new Result(q); + r.setCoverage(new Coverage(500, 1, true)); + + TimeTracker t = new TimeTracker(new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource( + new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + r.getElapsedTime().add(t); + r.getTemplating().setRenderer(d); + FastHit h = new FastHit("http://localhost/", .95); + h.setField("$a", "Hello, world."); + h.setField("b", "foo"); + r.hits().add(h); + HitGroup g = new HitGroup("usual"); + h = new FastHit("http://localhost/1", .90); + h.setField("c", "d"); + g.add(h); + r.hits().add(g); + HitGroup gg = new HitGroup("type grouphit"); + gg.types().add("grouphit"); + gg.setField("e", "f"); + r.hits().add(gg); + r.hits().addError(ErrorMessage.createInternalServerError("boom")); + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture<Boolean> f = d.render(bs, r, null, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + // TODO figure out a reasonably strict and reasonably flexible way to test + assertTrue(summary.length() > 1000); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java new file mode 100644 index 00000000000..a51dfc1b12f --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/rendering/XMLRendererTestCase.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.rendering; + +import static org.junit.Assert.*; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; + +import com.yahoo.search.handler.SearchHandler; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Coverage; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.statistics.ElapsedTimeTestCase; +import com.yahoo.search.statistics.TimeTracker; +import com.yahoo.search.statistics.ElapsedTimeTestCase.CreativeTimeSource; +import com.yahoo.search.statistics.ElapsedTimeTestCase.UselessSearcher; +import com.yahoo.text.Utf8; + +/** + * Test the XML renderer + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class XMLRendererTestCase { + + DefaultRenderer d; + + @Before + public void setUp() throws Exception { + d = new DefaultRenderer(); + d.init(); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetEncoding() { + assertEquals("utf-8", d.getEncoding()); + } + + @Test + public final void testGetMimeType() { + assertEquals("text/xml", d.getMimeType()); + } + + @Test + public final void testImplicitDefaultRender() throws Exception { + Query q = new Query("/?query=a&tracelevel=5&reportCoverage=true"); + q.getPresentation().setTiming(true); + Result r = new Result(q); + r.setCoverage(new Coverage(500, 1, true)); + + TimeTracker t = new TimeTracker(new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + ElapsedTimeTestCase.doInjectTimeSource(t, new CreativeTimeSource( + new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + r.getElapsedTime().add(t); + r.getTemplating().setRenderer(d); + FastHit h = new FastHit("http://localhost/", .95); + h.setField("$a", "Hello, world."); + h.setField("b", "foo"); + r.hits().add(h); + HitGroup g = new HitGroup("usual"); + h = new FastHit("http://localhost/1", .90); + h.setField("c", "d"); + g.add(h); + r.hits().add(g); + HitGroup gg = new HitGroup("type grouphit"); + gg.types().add("grouphit"); + gg.setField("e", "f"); + r.hits().add(gg); + r.hits().addError(ErrorMessage.createInternalServerError("boom")); + + ByteArrayOutputStream bs = new ByteArrayOutputStream(); + ListenableFuture<Boolean> f = d.render(bs, r, null, null); + assertTrue(f.get()); + String summary = Utf8.toString(bs.toByteArray()); + + assertEquals("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + + "<result total-hit-count=\"0\"", + summary.substring(0, 67) + ); + assertTrue(summary.contains("<meta type=\"context\">")); + assertTrue(summary.contains("<error code=\"18\">Internal server error.</error>")); + assertTrue(summary.contains("<hit type=\"grouphit\" relevancy=\"1.0\">")); + assertTrue(summary.contains("<hit type=\"summary\" relevancy=\"0.95\">")); + assertEquals(2, occurrences("<error ", summary)); + assertTrue(summary.length() > 1000); + } + + private int occurrences(String fragment, String string) { + int occurrences = 0; + int cursor = 0; + while ( -1 != (cursor = string.indexOf(fragment, cursor))) { + occurrences++; + cursor += fragment.length(); + } + return occurrences; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/DefaultErrorHitTestCase.java b/container-search/src/test/java/com/yahoo/search/result/DefaultErrorHitTestCase.java new file mode 100644 index 00000000000..582b8be1170 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/DefaultErrorHitTestCase.java @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result; + +import static org.junit.Assert.*; + +import java.util.Iterator; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * @author steinar + * @author bratseth + */ +public class DefaultErrorHitTestCase { + + private static final String SOURCE = "nalle"; + DefaultErrorHit de; + + @Before + public void setUp() throws Exception { + de = new DefaultErrorHit(SOURCE, ErrorMessage.createUnspecifiedError("DefaultErrorHitTestCase")); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testSetSourceTakeTwo() { + assertEquals(SOURCE, de.getSource()); + de.setSource(null); + assertNull(de.getSource()); + de.setSource("bamse"); + assertEquals("bamse", de.getSource()); + de.addError(ErrorMessage.createBackendCommunicationError("blblbl")); + final Iterator<ErrorMessage> errorIterator = de.errorIterator(); + assertEquals(SOURCE, errorIterator.next().getSource()); + assertEquals("bamse", errorIterator.next().getSource()); + } + + @Test + public final void testToString() { + assertEquals("Error: Source 'nalle': 5: Unspecified error: DefaultErrorHitTestCase", de.toString()); + } + + @Test + public final void testSetMainError() { + ErrorMessage e = ErrorMessage.createBackendCommunicationError("abc"); + assertNull(e.getSource()); + de.addError(e); + assertEquals(SOURCE, e.getSource()); + boolean caught = false; + try { + new DefaultErrorHit(SOURCE, null); + } catch (NullPointerException ex) { + caught = true; + } + assertTrue(caught); + + caught = false; + try { + de.addError(null); + } catch (NullPointerException ex) { + caught = true; + } + assertTrue(caught); + } + + @Test + public final void testAddError() { + ErrorMessage e = ErrorMessage + .createBackendCommunicationError("ljkhlkjh"); + assertNull(e.getSource()); + de.addError(e); + assertEquals(SOURCE, e.getSource()); + e = ErrorMessage.createBadRequest("kdjfhsdkfhj"); + de.addError(e); + int i = 0; + for (Iterator<ErrorMessage> errors = de.errorIterator(); errors + .hasNext(); errors.next()) { + ++i; + } + assertEquals(3, i); + } + + @Test + public final void testAddErrors() { + DefaultErrorHit other = new DefaultErrorHit("abc", + ErrorMessage.createBadRequest("sdasd")); + de.addErrors(other); + int i = 0; + for (Iterator<ErrorMessage> errors = de.errorIterator(); errors + .hasNext(); errors.next()) { + ++i; + } + assertEquals(2, i); + other = new DefaultErrorHit("abd", + ErrorMessage.createEmptyDocsums("uiyoiuy")); + other.addError(ErrorMessage.createNoAnswerWhenPingingNode("xzvczx")); + de.addErrors(other); + i = 0; + for (Iterator<ErrorMessage> errors = de.errorIterator(); errors + .hasNext(); errors.next()) { + ++i; + } + assertEquals(4, i); + } + + @Test + public final void testHasOnlyErrorCode() { + assertTrue(de.hasOnlyErrorCode(com.yahoo.container.protect.Error.UNSPECIFIED.code)); + assertFalse(de.hasOnlyErrorCode(com.yahoo.container.protect.Error.BACKEND_COMMUNICATION_ERROR.code)); + + de.addError(ErrorMessage.createUnspecifiedError("dsfsdfs")); + assertTrue(de.hasOnlyErrorCode(com.yahoo.container.protect.Error.UNSPECIFIED.code)); + assertEquals(com.yahoo.container.protect.Error.UNSPECIFIED.code, de.errors().iterator().next().getCode()); + + de.addError(ErrorMessage.createBackendCommunicationError("dsfsdfsd")); + assertFalse(de.hasOnlyErrorCode(com.yahoo.container.protect.Error.UNSPECIFIED.code)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/NanNumberTestCase.java b/container-search/src/test/java/com/yahoo/search/result/NanNumberTestCase.java new file mode 100644 index 00000000000..f6b2472cfb5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/NanNumberTestCase.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result; + +import static org.junit.Assert.*; + +import org.junit.Test; + +/** + * Integrity test for representation of undefined numeric field values. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class NanNumberTestCase { + + + @Test + public final void testIntValue() { + assertEquals(0, NanNumber.NaN.intValue()); + } + + @Test + public final void testLongValue() { + assertEquals(0L, NanNumber.NaN.longValue()); + } + + @Test + public final void testFloatValue() { + assertTrue(Float.isNaN(NanNumber.NaN.floatValue())); + } + + @Test + public final void testDoubleValue() { + assertTrue(Double.isNaN(NanNumber.NaN.doubleValue())); + } + + @Test + public final void testToString() { + assertEquals("", NanNumber.NaN.toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java b/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java new file mode 100644 index 00000000000..0e382e454b1 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/TemplatingTestCase.java @@ -0,0 +1,174 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.io.Writer; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import com.yahoo.search.rendering.Renderer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.base.Splitter; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.templates.UserTemplate; +import com.yahoo.prelude.templates.test.BoomTemplate; +import com.yahoo.search.Query; +import com.yahoo.search.Result; + +/** + * Control helper method for result rendering/result templates. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class TemplatingTestCase { + Result result; + + @Before + public void setUp() throws Exception { + Query q = new Query("/?query=a&presentation.format=nalle&offset=1&hits=5"); + result = new Result(q); + result.setTotalHitCount(1000L); + result.hits().add(new FastHit("http://localhost/1", .95)); + result.hits().add(new FastHit("http://localhost/2", .90)); + result.hits().add(new FastHit("http://localhost/3", .85)); + result.hits().add(new FastHit("http://localhost/4", .80)); + result.hits().add(new FastHit("http://localhost/5", .75)); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetFirstHitNo() { + assertEquals(2, result.getTemplating().getFirstHitNo()); + } + + @Test + public final void testGetNextFirstHitNo() { + assertEquals(7, result.getTemplating().getNextFirstHitNo()); + result.getQuery().setHits(6); + assertEquals(0, result.getTemplating().getNextFirstHitNo()); + } + + @Test + public final void testGetNextLastHitNo() { + assertEquals(11, result.getTemplating().getNextLastHitNo()); + result.getQuery().setHits(6); + assertEquals(0, result.getTemplating().getNextLastHitNo()); + } + + @Test + public final void testGetLastHitNo() { + assertEquals(6, result.getTemplating().getLastHitNo()); + } + + @Test + public final void testGetPrevFirstHitNo() { + assertEquals(1, result.getTemplating().getPrevFirstHitNo()); + } + + @Test + public final void testGetPrevLastHitNo() { + assertEquals(1, result.getTemplating().getPrevLastHitNo()); + } + + @Test + public final void testGetNextResultURL() { + String next = result.getTemplating().getNextResultURL(); + Set<String> expectedParameters = new HashSet<>(Arrays.asList(new String[] { + "hits=5", + "query=a", + "presentation.format=nalle", + "offset=6" + })); + Set<String> actualParameters = new HashSet<>(); + Splitter s = Splitter.on("&"); + for (String parameter : s.split(next.substring(next.indexOf('?') + 1))) { + actualParameters.add(parameter); + } + assertEquals(expectedParameters, actualParameters); + } + + @Test + public final void testGetPreviousResultURL() { + String previous = result.getTemplating().getPreviousResultURL(); + Set<String> expectedParameters = new HashSet<>(Arrays.asList(new String[] { + "hits=5", + "query=a", + "presentation.format=nalle", + "offset=0" + })); + Set<String> actualParameters = new HashSet<>(); + Splitter s = Splitter.on("&"); + for (String parameter : s.split(previous.substring(previous.indexOf('?') + 1))) { + actualParameters.add(parameter); + } + assertEquals(expectedParameters, actualParameters); + } + + @Test + public final void testGetCurrentResultURL() { + String previous = result.getTemplating().getCurrentResultURL(); + Set<String> expectedParameters = new HashSet<>(Arrays.asList(new String[] { + "hits=5", + "query=a", + "presentation.format=nalle", + "offset=1" + })); + Set<String> actualParameters = new HashSet<>(); + Splitter s = Splitter.on("&"); + for (String parameter : s.split(previous.substring(previous.indexOf('?') + 1))) { + actualParameters.add(parameter); + } + assertEquals(expectedParameters, actualParameters); + } + + @Test + public final void testGetTemplates() { + @SuppressWarnings({ "unchecked", "deprecation" }) + UserTemplate<Writer> t = result.getTemplating().getTemplates(); + assertEquals("default", t.getName()); + } + + @SuppressWarnings("deprecation") + @Test + public final void testSetTemplates() { + result.getTemplating().setTemplates(new BoomTemplate("gnuff", "text/plain", "ISO-8859-15")); + @SuppressWarnings("unchecked") + UserTemplate<Writer> t = result.getTemplating().getTemplates(); + assertEquals("gnuff", t.getName()); + } + + private static class TestRenderer extends Renderer { + + @Override + public void render(Writer writer, Result result) throws IOException { + } + + @Override + public String getEncoding() { + return null; + } + + @Override + public String getMimeType() { + return null; + } + } + + @SuppressWarnings("deprecation") + @Test + public final void testUsesDefaultTemplate() { + assertTrue(result.getTemplating().usesDefaultTemplate()); + result.getTemplating().setRenderer(new TestRenderer()); + assertFalse(result.getTemplating().usesDefaultTemplate()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java new file mode 100644 index 00000000000..35841a72428 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import java.io.IOException; + +import com.yahoo.prelude.hitfield.XMLString; +import com.yahoo.prelude.templates.test.TilingTestCase; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; + +/** + * @author bratseth + */ +public class ArrayOutputTestCase extends junit.framework.TestCase { + + public void testArrayOutput() throws IOException { + Result r=new Result(new Query("?query=ignored")); + Hit hit=new Hit("test"); + hit.setField("phone",new XMLString("\n <item>408-555-1234</item>" + "\n <item>408-555-5678</item>\n ")); + r.hits().add(hit); + + String rendered = TilingTestCase.getRendered(r); + String[] lines= rendered.split("\n"); + assertEquals(" <field name=\"phone\">",lines[4]); + assertEquals(" <item>408-555-1234</item>",lines[5]); + assertEquals(" <item>408-555-5678</item>",lines[6]); + assertEquals(" </field>",lines[7]); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java new file mode 100644 index 00000000000..efa01cc7c53 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/CoverageTestCase.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Coverage; + +/** + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class CoverageTestCase extends junit.framework.TestCase { + + public void testZeroCoverage() { + Coverage c = new Coverage(0L, 0, false, 0); + assertEquals(0, c.getResultPercentage()); + assertEquals(0, c.getResultSets()); + } + + public void testActiveCoverage() { + Coverage c = new Coverage(6, 5); + assertEquals(5, c.getActive()); + assertEquals(6, c.getDocs()); + + Coverage d = new Coverage(7, 6); + c.merge(d); + assertEquals(11, c.getActive()); + assertEquals(13, c.getDocs()); + } + + public void testDefaultCoverage() { + boolean create=true; + + Result r1=new Result(new Query()); + assertEquals(0,r1.getCoverage(create).getResultSets()); + Result r2=new Result(new Query()); + + r1.mergeWith(r2); + assertEquals(0,r1.getCoverage(create).getResultSets()); + } + + public void testDefaultSearchScenario() { + boolean create=true; + + Result federationSearcherResult=new Result(new Query()); + Result singleSourceResult=new Result(new Query()); + federationSearcherResult.mergeWith(singleSourceResult); + assertNull(federationSearcherResult.getCoverage(!create)); + assertEquals(0,federationSearcherResult.getCoverage(create).getResultSets()); + } + + public void testRequestingCoverageSearchScenario() { + boolean create=true; + + Result federationSearcherResult=new Result(new Query()); + Result singleSourceResult=new Result(new Query()); + singleSourceResult.setCoverage(new Coverage(10,1,true)); + federationSearcherResult.mergeWith(singleSourceResult); + assertEquals(1,federationSearcherResult.getCoverage(create).getResultSets()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/DeepHitIteratorTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/DeepHitIteratorTestCase.java new file mode 100644 index 00000000000..386e04ba943 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/DeepHitIteratorTestCase.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import com.yahoo.search.result.DeepHitIterator; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +/** + * Ensure that the {@link DeepHitIterator} works as intended. + * + * @author havardpe + */ +public class DeepHitIteratorTestCase extends junit.framework.TestCase { + + public void testEmpty() { + HitGroup hits = new HitGroup(); + Iterator<Hit> it = hits.deepIterator(); + assertFalse(it.hasNext()); + try { + it.next(); + fail(); + } catch (NoSuchElementException e) { + // regular iterator behavior + } + } + + public void testRemove() { + HitGroup hits = new HitGroup(); + hits.add(new Hit("foo")); + hits.add(new Hit("bar")); + + Iterator<Hit> it = hits.deepIterator(); + try { + it.remove(); + fail(); + } catch (IllegalStateException e) { + // need to call next() first + } + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertTrue(it.hasNext()); + try { + it.remove(); + fail(); + } catch (IllegalStateException e) { + // prefetch done + } + assertEquals("bar", it.next().getId().toString()); + it.remove(); // no prefetch done + assertFalse(it.hasNext()); + } + + public void testShallow() { + HitGroup hits = new HitGroup(); + hits.add(new Hit("foo")); + hits.add(new Hit("bar")); + hits.add(new Hit("baz")); + + Iterator<Hit> it = hits.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("bar", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("baz", it.next().getId().toString()); + assertFalse(it.hasNext()); + } + + public void testDeep() { + HitGroup grandParent = new HitGroup(); + grandParent.add(new Hit("a")); + HitGroup parent = new HitGroup(); + parent.add(new Hit("b")); + HitGroup child = new HitGroup(); + child.add(new Hit("c")); + HitGroup grandChild = new HitGroup(); + grandChild.add(new Hit("d")); + child.add(grandChild); + child.add(new Hit("e")); + parent.add(child); + parent.add(new Hit("f")); + grandParent.add(parent); + grandParent.add(new Hit("g")); + + Iterator<Hit> it = grandParent.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("a", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("b", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("c", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("d", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("e", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("f", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("g", it.next().getId().toString()); + assertFalse(it.hasNext()); + } + + public void testFirstHitIsGroup() { + HitGroup root = new HitGroup(); + HitGroup group = new HitGroup(); + group.add(new Hit("foo")); + root.add(group); + root.add(new Hit("bar")); + + Iterator<Hit> it = root.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("bar", it.next().getId().toString()); + assertFalse(it.hasNext()); + } + + public void testSecondHitIsGroup() { + HitGroup root = new HitGroup(); + root.add(new Hit("foo")); + HitGroup group = new HitGroup(); + group.add(new Hit("bar")); + root.add(group); + + Iterator<Hit> it = root.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertTrue(it.hasNext()); + assertEquals("bar", it.next().getId().toString()); + assertFalse(it.hasNext()); + } + + public void testOrder() { + HitGroup root = new HitGroup(); + MyHitGroup group = new MyHitGroup(); + group.add(new Hit("foo")); + root.add(group); + + Iterator<Hit> it = root.deepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertEquals(Boolean.TRUE, group.ordered); + assertFalse(it.hasNext()); + + it = root.unorderedDeepIterator(); + assertTrue(it.hasNext()); + assertEquals("foo", it.next().getId().toString()); + assertEquals(Boolean.FALSE, group.ordered); + assertFalse(it.hasNext()); + } + + @SuppressWarnings("serial") + private static class MyHitGroup extends HitGroup { + + Boolean ordered = null; + + @Override + public Iterator<Hit> iterator() { + ordered = Boolean.TRUE; + return super.iterator(); + } + + @Override + public Iterator<Hit> unorderedIterator() { + ordered = Boolean.FALSE; + return super.unorderedIterator(); + } + } +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/DefaultErrorHitTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/DefaultErrorHitTestCase.java new file mode 100644 index 00000000000..2935c826539 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/DefaultErrorHitTestCase.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import com.yahoo.prelude.templates.SearchRendererAdaptor; +import com.yahoo.search.result.DefaultErrorHit; +import com.yahoo.search.result.ErrorMessage; + +import java.io.IOException; +import java.io.StringWriter; + +/** + * @author bratseth + */ +public class DefaultErrorHitTestCase extends junit.framework.TestCase { + + @SuppressWarnings("null") + public void testErrorHitRenderingWithException() throws IOException { + NullPointerException cause=null; + try { + Object a=null; + a.toString(); + } + catch (NullPointerException e) { + cause=e; + } + StringWriter w=new StringWriter(); + SearchRendererAdaptor.simpleRenderDefaultErrorHit(w, new DefaultErrorHit("test", new ErrorMessage(79, "Myerror", "Mydetail", cause))); + String sep = System.getProperty("line.separator"); + assertEquals( + "<errordetails>\n" + + " <error source=\"test\" error=\"Myerror\" code=\"79\">Mydetail\n" + + " <cause>\n" + + "java.lang.NullPointerException" + sep + + "\tat " + ,w.toString().substring(0, 119+sep.length())); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java new file mode 100644 index 00000000000..9e16b7312eb --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/FillingTestCase.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +/** + * @author bratseth + */ +public class FillingTestCase extends junit.framework.TestCase { + + public void testFillingAPIConsistency() { + HitGroup group=new HitGroup(); + group.add(new Hit("hit:1")); + group.add(new Hit("hit:2")); + assertTrue(group.isFilled("summary")); + } + + public void testFillingAPIConsistencyTwoPhase() { + HitGroup group=new HitGroup(); + group.add(createNonFilled("hit:1")); + group.add(createNonFilled("hit:2")); + assertFalse(group.isFilled("summary")); + fillHitsIn(group, "summary"); + group.analyze(); + assertTrue(group.isFilled("summary")); // consistent again + } + + public void testFillingAPIConsistencyThreePhase() { + HitGroup group=new HitGroup(); + group.add(createNonFilled("hit:1")); + group.add(createNonFilled("hit:2")); + assertFalse(group.isFilled("summary")); + assertFalse(group.isFilled("otherSummary")); + fillHitsIn(group, "otherSummary"); + group.analyze(); + assertFalse(group.isFilled("summary")); + assertTrue(group.isFilled("otherSummary")); + fillHitsIn(group, "summary"); + assertTrue(group.isFilled("otherSummary")); + group.analyze(); + assertTrue(group.isFilled("summary")); // consistent again + assertTrue(group.isFilled("otherSummary")); + } + + private Hit createNonFilled(String id) { + Hit hit=new Hit(id); + hit.setFillable(); + return hit; + } + + private void fillHitsIn(HitGroup group,String summary) { + for (Hit hit : group.asList()) { + if (hit.isMeta()) continue; + hit.setFilled(summary); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java new file mode 100644 index 00000000000..c2d5e73fb97 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/result/test/HitGroupTestCase.java @@ -0,0 +1,189 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.result.test; + +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +import java.util.Arrays; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class HitGroupTestCase extends junit.framework.TestCase { + + public void testStringStripping() { + assertEquals("avabarne", Hit.stripCharacter('j', "javabjarne")); + assertEquals("", Hit.stripCharacter('j', "")); + assertEquals("", Hit.stripCharacter('j', "j")); + assertEquals("frank", Hit.stripCharacter('j', "frank")); + assertEquals("foo", Hit.stripCharacter('j', "fooj")); + assertEquals("", Hit.stripCharacter('j', "jjjjj")); + } + + public void testRecursiveGet() { + // Level 1 + HitGroup g1=new HitGroup(); + g1.add(new Hit("1")); + + // Level 2 + HitGroup g1_1=new HitGroup(); + g1_1.add(new Hit("1.1")); + g1.add(g1_1); + + HitGroup g1_2=new HitGroup(); + g1_2.add(new Hit("1.2")); + g1.add(g1_2); + + // Level 3 + HitGroup g1_1_1=new HitGroup(); + g1_1_1.add(new Hit("1.1.1")); + g1_1.add(g1_1_1); + + HitGroup g1_1_2=new HitGroup(); + g1_1_2.add(new Hit("1.1.2")); + g1_1.add(g1_1_2); + + HitGroup g1_2_1=new HitGroup(); + g1_2_1.add(new Hit("1.2.1")); + g1_2.add(g1_2_1); + + HitGroup g1_2_2=new HitGroup(); + g1_2_2.add(new Hit("1.2.2")); + g1_2.add(g1_2_2); + + // Level 4 + HitGroup g1_1_1_1=new HitGroup(); + g1_1_1_1.add(new Hit("1.1.1.1")); + g1_1_1.add(g1_1_1_1); + + assertNotNull(g1.get("1")); + assertNotNull(g1.get("1.1")); + assertNotNull(g1.get("1.2")); + assertNotNull(g1.get("1.1.1")); + assertNotNull(g1.get("1.1.2")); + assertNotNull(g1.get("1.2.1")); + assertNotNull(g1.get("1.2.2")); + assertNotNull(g1.get("1.1.1.1")); + + assertNotNull(g1.get("1",-1)); + assertNotNull(g1.get("1.1",-1)); + assertNotNull(g1.get("1.2",-1)); + assertNotNull(g1.get("1.1.1",-1)); + assertNotNull(g1.get("1.1.2",-1)); + assertNotNull(g1.get("1.2.1",-1)); + assertNotNull(g1.get("1.2.2",-1)); + assertNotNull(g1.get("1.1.1.1",-1)); + + assertNotNull(g1.get("1",0)); + assertNull(g1.get("1.1",0)); + assertNull(g1.get("1.2",0)); + assertNull(g1.get("1.1.1",0)); + assertNull(g1.get("1.1.2",0)); + assertNull(g1.get("1.2.1",0)); + assertNull(g1.get("1.2.2",0)); + assertNull(g1.get("1.1.1.1",0)); + + assertNotNull(g1.get("1",1)); + assertNotNull(g1.get("1.1",1)); + assertNotNull(g1.get("1.2",1)); + assertNull(g1.get("1.1.1",1)); + assertNull(g1.get("1.1.2",1)); + assertNull(g1.get("1.2.1",1)); + assertNull(g1.get("1.2.2",1)); + assertNull(g1.get("1.1.1.1",1)); + + assertNotNull(g1.get("1",2)); + assertNotNull(g1.get("1.1",2)); + assertNotNull(g1.get("1.2",2)); + assertNotNull(g1.get("1.1.1",2)); + assertNotNull(g1.get("1.1.2",2)); + assertNotNull(g1.get("1.2.1",2)); + assertNotNull(g1.get("1.2.2",2)); + assertNull(g1.get("1.1.1.1",2)); + + assertNotNull(g1.get("1.1.1.1",3)); + + assertNull(g1.get("3",2)); + } + + public void testThatHitGroupIsUnFillable() { + HitGroup hg = new HitGroup("test"); + { + Hit hit = new Hit("http://nalle.balle/1.html", 832); + hit.setField("url", "http://nalle.balle/1.html"); + hit.setField("clickurl", "javascript:openWindow('http://www.foo');"); + hit.setField("attributes", Arrays.asList("typevideo")); + hg.add(hit); + } + { + Hit hit = new Hit("http://nalle.balle/2.html", 442); + hit.setField("url", "http://nalle.balle/2.html"); + hit.setField("clickurl", ""); + hit.setField("attributes", Arrays.asList("typevideo")); + hg.add(hit); + } + assertFalse(hg.isFillable()); + assertTrue(hg.isFilled("anyclass")); + assertNull(hg.getFilled()); + } + + public void testThatHitGroupIsFillable() { + HitGroup hg = new HitGroup("test"); + { + Hit hit = new Hit("http://nalle.balle/1.html", 832); + hit.setField("url", "http://nalle.balle/1.html"); + hit.setField("clickurl", "javascript:openWindow('http://www.foo');"); + hit.setField("attributes", Arrays.asList("typevideo")); + hit.setFillable(); + hg.add(hit); + } + { + Hit hit = new Hit("http://nalle.balle/2.html", 442); + hit.setField("url", "http://nalle.balle/2.html"); + hit.setField("clickurl", ""); + hit.setField("attributes", Arrays.asList("typevideo")); + hit.setFillable(); + hg.add(hit); + } + assertTrue(hg.isFillable()); + assertFalse(hg.isFilled("anyclass")); + assertTrue(hg.getFilled().isEmpty()); + } + + public void testThatHitGroupIsFillableAfterFillableChangeunderTheHood() { + HitGroup hg = new HitGroup("test"); + { + Hit hit = new Hit("http://nalle.balle/1.html", 832); + hit.setField("url", "http://nalle.balle/1.html"); + hit.setField("clickurl", "javascript:openWindow('http://www.foo');"); + hit.setField("attributes", Arrays.asList("typevideo")); + hg.add(hit); + } + { + Hit hit = new Hit("http://nalle.balle/2.html", 442); + hit.setField("url", "http://nalle.balle/2.html"); + hit.setField("clickurl", ""); + hit.setField("attributes", Arrays.asList("typevideo")); + hg.add(hit); + } + assertFalse(hg.isFillable()); + assertTrue(hg.isFilled("anyclass")); + + for (Hit h : hg.asList()) { + h.setFillable(); + } + + HitGroup toplevel = new HitGroup("toplevel"); + toplevel.add(hg); + + assertTrue(toplevel.isFillable()); + assertNotNull(toplevel.getFilled()); + assertFalse(toplevel.isFilled("anyclass")); + + assertTrue(hg.isFillable()); + assertNotNull(hg.getFilled()); + assertFalse(hg.isFilled("anyclass")); + assertTrue(hg.getFilled().isEmpty()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java new file mode 100644 index 00000000000..9066df45309 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/DependencyConfigTestCase.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.dependencies.Provides; +import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChainRegistry; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * @author tonytv + */ +public class DependencyConfigTestCase { + + private static HandlersConfigurerTestWrapper configurer; + + private static SearchChainRegistry registry; + + public static final String root = "src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig"; + + @BeforeClass + public static void createComponentsConfig() throws IOException { + SearchChainConfigurerTestCase. + createComponentsConfig(root + "/chains.cfg", root + "/handlers.cfg", root + "/components.cfg"); + setUp(); + } + + @AfterClass + public static void removeComponentsConfig() throws IOException { + new File(root + "/components.cfg").delete(); + tearDown(); + } + + public static void setUp() { + String configId = "dir:" + root; + configurer = new HandlersConfigurerTestWrapper(configId); + registry=((SearchHandler) configurer.getRequestHandlerRegistry().getComponent("com.yahoo.search.handler.SearchHandler")).getSearchChainRegistry(); + } + + public static void tearDown() { + configurer.shutdown(); + } + + @Provides("P") + @Before("B") + @After("A") + public static class Searcher1 extends Searcher { + + public Result search(Query query,Execution execution) { + return execution.search(query); + } + + } + + @Test + public void test() { + Dependencies dependencies = registry.getSearcherRegistry().getComponent(Searcher1.class.getName()).getDependencies(); + + assertTrue(dependencies.provides().containsAll(Arrays.asList("P", "P1", "P2", Searcher1.class.getSimpleName()))); + assertTrue(dependencies.before().containsAll(Arrays.asList("B", "B1", "B2"))); + assertTrue(dependencies.after().containsAll(Arrays.asList("A", "A1", "A2"))); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java new file mode 100644 index 00000000000..18073a1cedd --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/SearchChainConfigurerTestCase.java @@ -0,0 +1,302 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test; + +import com.yahoo.config.search.IntConfig; +import com.yahoo.config.search.StringConfig; +import com.yahoo.container.config.testutil.TestUtil; +import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.handler.SearchHandler; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.SearcherRegistry; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.*; +import java.util.*; + +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.*; + +/** + * @author bratseth + * @author gjoranv + */ +public class SearchChainConfigurerTestCase { + + private static Random random = new Random(1); + private static String topCfgDir = System.getProperty("java.io.tmpdir") + File.separator + + "SearchChainConfigurerTestCase" + File.separator; + + private static final String testDir = "src/test/java/com/yahoo/search/searchchain/config/test/"; + + + public void cleanup(File cfgDir) { + if (cfgDir.exists()) { + for (File f : cfgDir.listFiles()) { + f.delete(); + } + cfgDir.delete(); + } + } + + @BeforeClass + public static void createDefaultComponentsConfigs() throws IOException { + createComponentsConfig(testDir + "chains.cfg", testDir + "handlers.cfg", testDir + "components.cfg"); + } + + @AfterClass + public static void removeDefaultComponentsConfigs() throws IOException { + new File(testDir + "components.cfg").delete(); + } + + private SearchChainRegistry getSearchChainRegistryFrom(HandlersConfigurerTestWrapper configurer) { + return ((SearchHandler)configurer.getRequestHandlerRegistry(). + getComponent("com.yahoo.search.handler.SearchHandler")).getSearchChainRegistry(); + } + + @Test + public synchronized void testConfiguration() throws Exception { + HandlersConfigurerTestWrapper configurer = new HandlersConfigurerTestWrapper("dir:" + testDir); + + SearchChain simple=getSearchChainRegistryFrom(configurer).getComponent("simple"); + assertNotNull(simple); + assertThat(getSearcherNumbers(simple), is(Arrays.asList(1, 2, 3))); + + SearchChain child1=getSearchChainRegistryFrom(configurer).getComponent("child:1"); + assertThat(getSearcherNumbers(child1), is(Arrays.asList(1, 2, 4, 5, 7, 8))); + + SearchChain child2=getSearchChainRegistryFrom(configurer).getComponent("child"); + assertThat(getSearcherNumbers(child2), is(Arrays.asList(3, 6, 7, 9))); + + // Verify successful loading of an explicitly declared searcher that takes no user-defined configs. + //assertNotNull(SearchChainRegistry.get().getSearcherRegistry().getComponent + // ("com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher")); + configurer.shutdown(); + } + + private List<Integer> getSearcherNumbers(SearchChain chain) { + List<Integer> numbers = new ArrayList<>(); + for (int i=0; i<chain.searchers().size(); i++) { + String prefix=TestSearcher.class.getName(); + assertTrue(chain.searchers().get(i).getId().getName().startsWith(prefix)); + int value = Integer.parseInt(chain.searchers().get(i).getId().getName().substring(prefix.length())); + numbers.add(value); + } + Collections.sort(numbers); + return numbers; + } + + public static abstract class TestSearcher extends Searcher { + public @Override Result search(Query query, Execution execution) { + return execution.search(query); + } + } + public static final class TestSearcher1 extends TestSearcher {} + public static final class TestSearcher2 extends TestSearcher {} + public static final class TestSearcher3 extends TestSearcher {} + public static final class TestSearcher4 extends TestSearcher {} + public static final class TestSearcher5 extends TestSearcher {} + public static final class TestSearcher6 extends TestSearcher {} + public static final class TestSearcher7 extends TestSearcher {} + public static final class TestSearcher8 extends TestSearcher {} + public static final class TestSearcher9 extends TestSearcher {} + public static final class DeclaredTestSearcher extends TestSearcher {} + + @Test + public void testConfigurableSearcher() { + HandlersConfigurerTestWrapper configurer=new HandlersConfigurerTestWrapper("dir:" + testDir); + + SearchChain configurable = getSearchChainRegistryFrom(configurer).getComponent("configurable"); + assertNotNull(configurable); + + Searcher s = configurable.searchers().get(0); + assertThat(s, instanceOf(ConfigurableSearcher.class)); + ConfigurableSearcher searcher = (ConfigurableSearcher)s; + assertThat("Value from int.cfg file", searcher.intConfig.intVal(), is(7)); + assertThat("Value from string.cfg file", searcher.stringConfig.stringVal(), + is("com.yahoo.search.searchchain.config.test")); + configurer.shutdown(); + } + + /** + * Verifies that only searchers with updated config are re-instantiated after a config update + * that does not contain any bootstrap configs. + */ + @Test + public void testSearcherConfigUpdate() throws IOException, InterruptedException { + File cfgDir = getCfgDir(); + copyFile(testDir + "handlers.cfg", cfgDir + "/handlers.cfg"); + copyFile(testDir + "qr-search.cfg", cfgDir + "/qr-search.cfg"); + copyFile(testDir + "qr-searchers.cfg", cfgDir + "/qr-searchers.cfg"); + copyFile(testDir + "index-info.cfg", cfgDir + "/index-info.cfg"); + copyFile(testDir + "specialtokens.cfg", cfgDir + "/specialtokens.cfg"); + copyFile(testDir + "three-searchers.cfg", cfgDir + "/chains.cfg"); + createComponentsConfig(testDir + "three-searchers.cfg", testDir + "handlers.cfg", cfgDir + "/components.cfg"); + printFile(new File(cfgDir + "/int.cfg"), "intVal 16\n"); + printFile(new File(cfgDir + "/string.cfg"), "stringVal \"testSearcherConfigUpdate\"\n"); + + HandlersConfigurerTestWrapper configurer = new HandlersConfigurerTestWrapper("dir:" + cfgDir); + SearcherRegistry searchers = getSearchChainRegistryFrom(configurer).getSearcherRegistry(); + assertThat(searchers.getComponentCount(), is(3)); + + IntSearcher intSearcher = (IntSearcher)searchers.getComponent(IntSearcher.class.getName()); + assertThat(intSearcher.intConfig.intVal(), is(16)); + StringSearcher stringSearcher = (StringSearcher)searchers.getComponent(StringSearcher.class.getName()); + DeclaredTestSearcher noConfigSearcher = + (DeclaredTestSearcher)searchers.getComponent(DeclaredTestSearcher.class.getName()); + + // Update int config for IntSearcher, + printFile(new File(cfgDir + "/int.cfg"), "intVal 17\n"); + configurer.reloadConfig(); + + // Registry is rebuilt + assertThat(getSearchChainRegistryFrom(configurer).getSearcherRegistry(), not(searchers)); + searchers = getSearchChainRegistryFrom(configurer).getSearcherRegistry(); + assertThat(searchers.getComponentCount(), is(3)); + + // Searcher with updated config is re-instantiated. + IntSearcher intSearcher2 = (IntSearcher)searchers.getComponent(IntSearcher.class.getName()); + assertThat(intSearcher2, not(sameInstance(intSearcher))); + assertThat(intSearcher2.intConfig.intVal(), is(17)); + + // Searchers with unchanged config (or that takes no config) are the same as before. + Searcher s = searchers.getComponent(DeclaredTestSearcher.class.getName()); + assertThat((DeclaredTestSearcher)s, sameInstance(noConfigSearcher)); + s = searchers.getComponent(StringSearcher.class.getName()); + assertThat((StringSearcher)s, sameInstance(stringSearcher)); + + configurer.shutdown(); + cleanup(cfgDir); + } + + /** + * Updates the chains config, while the searcher configs are unchanged. + * Verifies that a new searcher that was not in the old config is instantiated, + * and that a searcher that has been removed from the configuration is not in the new registry. + */ + @Test + public void testChainsConfigUpdate() throws IOException, InterruptedException { + File cfgDir = getCfgDir(); + copyFile(testDir + "handlers.cfg", cfgDir + "/handlers.cfg"); + copyFile(testDir + "qr-search.cfg", cfgDir + "/qr-search.cfg"); + copyFile(testDir + "qr-searchers.cfg", cfgDir + "/qr-searchers.cfg"); + copyFile(testDir + "index-info.cfg", cfgDir + "/index-info.cfg"); + copyFile(testDir + "specialtokens.cfg", cfgDir + "/specialtokens.cfg"); + copyFile(testDir + "chainsConfigUpdate_1.cfg", cfgDir + "/chains.cfg"); + createComponentsConfig(testDir + "chainsConfigUpdate_1.cfg", testDir + "handlers.cfg", cfgDir + "/components.cfg"); + + HandlersConfigurerTestWrapper configurer = new HandlersConfigurerTestWrapper("dir:" + cfgDir); + + SearchChainRegistry scReg = getSearchChainRegistryFrom(configurer); + SearcherRegistry searchers = scReg.getSearcherRegistry(); + assertThat(searchers.getComponentCount(), is(2)); + assertThat(searchers.getComponent(IntSearcher.class.getName()), instanceOf(IntSearcher.class)); + assertThat(searchers.getComponent(StringSearcher.class.getName()), instanceOf(StringSearcher.class)); + assertThat(searchers.getComponent(ConfigurableSearcher.class.getName()), nullValue()); + assertThat(searchers.getComponent(DeclaredTestSearcher.class.getName()), nullValue()); + + IntSearcher intSearcher = (IntSearcher)searchers.getComponent(IntSearcher.class.getName()); + + // Update chains config + copyFile(testDir + "chainsConfigUpdate_2.cfg", cfgDir + "/chains.cfg"); + createComponentsConfig(testDir + "chainsConfigUpdate_2.cfg", testDir + "handlers.cfg", cfgDir + "/components.cfg"); + configurer.reloadConfig(); + + assertThat(getSearchChainRegistryFrom(configurer), not(scReg)); + + // In the new registry, the correct searchers are removed and added + assertThat(getSearchChainRegistryFrom(configurer).getSearcherRegistry(), not(searchers)); + searchers = getSearchChainRegistryFrom(configurer).getSearcherRegistry(); + assertThat(searchers.getComponentCount(), is(3)); + assertThat((IntSearcher)searchers.getComponent(IntSearcher.class.getName()), sameInstance(intSearcher)); + assertThat(searchers.getComponent(ConfigurableSearcher.class.getName()), instanceOf(ConfigurableSearcher.class)); + assertThat(searchers.getComponent(DeclaredTestSearcher.class.getName()), instanceOf(DeclaredTestSearcher.class)); + assertThat(searchers.getComponent(StringSearcher.class.getName()), nullValue()); + configurer.shutdown(); + cleanup(cfgDir); + } + + public static class ConfigurableSearcher extends Searcher { + IntConfig intConfig; + StringConfig stringConfig; + + public ConfigurableSearcher(IntConfig intConfig) { + this.intConfig = intConfig; + } + public ConfigurableSearcher(IntConfig intConfig, StringConfig stringConfig) { + this.intConfig = intConfig; + this.stringConfig = stringConfig; + } + public @Override Result search(Query query, Execution execution) { + return execution.search(query); + } + } + + public static class IntSearcher extends Searcher { + IntConfig intConfig; + public IntSearcher(IntConfig intConfig) { + this.intConfig = intConfig; + } + public @Override Result search(Query query, Execution execution) { + return execution.search(query); + } + } + + public static class StringSearcher extends Searcher { + StringConfig stringConfig; + public StringSearcher(StringConfig stringConfig) { + this.stringConfig = stringConfig; + } + public @Override Result search(Query query, Execution execution) { + return execution.search(query); + } + } + + + //// Helper methods + + public static void printFile(File f, String content) throws IOException, InterruptedException { + OutputStream out = new FileOutputStream(f); + out.write(content.getBytes()); + out.close(); + + } + + /** + * Copies src file to dst file. If the dst file does not exist, it is created. + */ + public static void copyFile(String srcName, String dstName) throws IOException { + InputStream src = new FileInputStream(new File(srcName)); + OutputStream dst = new FileOutputStream(new File(dstName)); + byte[] buf = new byte[1024]; + int len; + while ((len = src.read(buf)) > 0) { + dst.write(buf, 0, len); + } + src.close(); + dst.close(); + } + + public static File getCfgDir() { + String token = Long.toHexString(random.nextLong()); + File cfgDir = new File(topCfgDir + File.separator + token + File.separator); + cfgDir.mkdirs(); + return cfgDir; + } + + /** + * Copies the ids from the 'search' array in chains to a 'components' array in a new components file. + * Also adds the default SearchHandler. + */ + public static void createComponentsConfig(String chainsFile, String handlersFile, String componentsFile) throws IOException { + TestUtil.createComponentsConfig(handlersFile, componentsFile, "handler"); + TestUtil.createComponentsConfig(chainsFile, componentsFile, "components", true); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chains.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chains.cfg new file mode 100644 index 00000000000..4c2b8b0ea27 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chains.cfg @@ -0,0 +1,56 @@ +chains[8] +chains[0].id simple +chains[0].components[3] +chains[0].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +chains[0].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher2 +chains[0].components[2] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher3 +chains[1].id mother:1.1 +chains[1].components[2] +chains[1].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +chains[1].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher2 +chains[2].id mother:1.2 +chains[2].components[2] +chains[2].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +chains[2].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher3 +chains[3].id father:2 +chains[3].components[2] +chains[3].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher4 +chains[3].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher6 +chains[4].id father:1 +chains[4].components[2] +chains[4].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher4 +chains[4].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher5 +chains[5].id child:1 +chains[5].components[2] +chains[5].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher7 +chains[5].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher8 +chains[5].inherits[2] +chains[5].inherits[0] mother:1.1 +chains[5].inherits[1] father:1 +chains[6].id child:2 +chains[6].components[2] +chains[6].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher7 +chains[6].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher9 +chains[6].inherits[2] +chains[6].inherits[0] mother +chains[6].inherits[1] father +chains[6].excludes[2] +chains[6].excludes[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +chains[6].excludes[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher4 +chains[7].id configurable +chains[7].components[1] +chains[7].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$ConfigurableSearcher + +components[11] +components[0].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$ConfigurableSearcher +components[1].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher +components[2].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher1 +components[3].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher2 +components[4].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher3 +components[5].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher4 +components[6].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher5 +components[7].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher6 +components[8].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher7 +components[9].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher8 +components[10].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$TestSearcher9 + diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_1.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_1.cfg new file mode 100755 index 00000000000..1c8c83cef10 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_1.cfg @@ -0,0 +1,8 @@ +chains[1] +chains[0].id test-chains-config-update +chains[0].components[2] +chains[0].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +chains[0].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$StringSearcher +components[2] +components[0].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +components[1].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$StringSearcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_2.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_2.cfg new file mode 100755 index 00000000000..9717f7b4ee5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/chainsConfigUpdate_2.cfg @@ -0,0 +1,10 @@ +chains[1] +chains[0].id test-chains-config-update +chains[0].components[3] +chains[0].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +chains[0].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$ConfigurableSearcher +chains[0].components[2] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher +components[3] +components[0].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +components[1].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$ConfigurableSearcher +components[2].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/chains.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/chains.cfg new file mode 100644 index 00000000000..bb4c2919bec --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/chains.cfg @@ -0,0 +1,20 @@ +chains[1] +chains[0].id "default" +chains[0].phases[2] +chains[0].phases[0].id "phase1" +chains[0].phases[1].id "phase2" +chains[0].phases[1].before[1] +chains[0].phases[1].before[0] "phase1" +chains[0].components[1] +chains[0].components[0] "com.yahoo.search.searchchain.config.test.DependencyConfigTestCase$Searcher1" +components[1] +components[0].id "com.yahoo.search.searchchain.config.test.DependencyConfigTestCase$Searcher1" +components[0].dependencies.provides[2] +components[0].dependencies.provides[0] "P1" +components[0].dependencies.provides[1] "P2" +components[0].dependencies.before[2] +components[0].dependencies.before[0] "B1" +components[0].dependencies.before[1] "B2" +components[0].dependencies.after[2] +components[0].dependencies.after[0] "A1" +components[0].dependencies.after[1] "A2" diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/index-info.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/index-info.cfg diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-search.cfg diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/dependencyConfig/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/implicitDependencies.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/implicitDependencies.cfg new file mode 100644 index 00000000000..d9838a95665 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/implicitDependencies.cfg @@ -0,0 +1,14 @@ +chains[1] +chains[0].id default +chains[0].components[2] +chains[0].components[0].id com.yahoo.search.searchchain.config.test.ImplicitDependenciesTestCase$First +chains[0].components[1].id com.yahoo.search.searchchain.config.test.ImplicitDependenciesTestCase$Second + +components[2] +components[0].id PoSearcher +components[0].classid com.yahoo.pageopt.system.PoSearcher +components[1].id ExampleSearcher +components[1].classid com.yahoo.pageopt.searcher.ExampleSearcher + +components[1].dependencies.after[1] +components[1].dependencies.after[0] PoSearcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/index-info.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/index-info.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/index-info.cfg diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/int.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/int.cfg new file mode 100644 index 00000000000..379e768d6f3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/int.cfg @@ -0,0 +1 @@ +intVal 7 diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-logging.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-logging.cfg new file mode 100644 index 00000000000..f514ae59a37 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-logging.cfg @@ -0,0 +1 @@ +speciallog[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-search.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-search.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-search.cfg diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Manifest.MF new file mode 100644 index 00000000000..09956bb2aef --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +Bundle-Version: 2.1 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Searcher1.java.text b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Searcher1.java.text new file mode 100644 index 00000000000..ecbedc85875 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.1/Searcher1.java.text @@ -0,0 +1,22 @@ +package com.yahoo.search.searchchain.config.test.searcher1; + +import com.yahoo.search.Searcher; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; + +/** + * @author bratseth + */ +public class Searcher1 extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:searcher1:2.1")); + return result; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Manifest.MF new file mode 100644 index 00000000000..719fe7a81ba --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +Bundle-Version: 2.2 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Searcher1.java.text b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Searcher1.java.text new file mode 100644 index 00000000000..413575aca39 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2.2/Searcher1.java.text @@ -0,0 +1,22 @@ +package com.yahoo.search.searchchain.config.test.searcher1; + +import com.yahoo.search.Searcher; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; + +/** + * @author bratseth + */ +public class Searcher1 extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:searcher1:2.2")); + return result; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Manifest.MF new file mode 100644 index 00000000000..9e7835f6ee2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +Bundle-Version: 2 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Searcher1.java.text b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Searcher1.java.text new file mode 100644 index 00000000000..29e5fb7697a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1-2/Searcher1.java.text @@ -0,0 +1,22 @@ +package com.yahoo.search.searchchain.config.test.searcher1; + +import com.yahoo.search.Searcher; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; + +/** + * @author bratseth + */ +public class Searcher1 extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:searcher1:2")); + return result; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Manifest.MF new file mode 100644 index 00000000000..70447069705 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +Bundle-Version: 0 +Bundle-Vendor: Yahoo! +Import-Package: com.yahoo.prelude, + org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Searcher1.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Searcher1.java new file mode 100644 index 00000000000..5c42629524b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher1/Searcher1.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.searcher1; + +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class Searcher1 extends Searcher { + + public @Override + Result search(Query query,Execution execution) { + Result result=execution.search(query); + ErrorMessage.createErrorInPluginSearcher("nop"); // Check that we may access legacy packages + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:searcher1:0")); + return result; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Manifest.MF new file mode 100644 index 00000000000..972c3090b8d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: HelloWorld +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.searcher2.Searcher2 +Bundle-Version: 1.0.0 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher2 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Searcher2.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Searcher2.java new file mode 100644 index 00000000000..942bb2fe97a --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/searcher2/Searcher2.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.searcher2; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class Searcher2 extends Searcher { + + public Result search(Query query, Execution execution) { + return execution.search(query); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/string.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/string.cfg new file mode 100644 index 00000000000..af532c6d565 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/string.cfg @@ -0,0 +1 @@ +stringVal "com.yahoo.search.searchchain.config.test" diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/chains.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/chains.cfg new file mode 100644 index 00000000000..609c4708306 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/chains.cfg @@ -0,0 +1,27 @@ +chains[3] +chains[0].id classInstances +chains[0].components[3] +chains[0].components[0] class1-instance1 +chains[0].components[1] class1-instance2 +chains[0].components[2] class2-instance2 +chains[1].id osgiInstances +chains[1].components[3] +chains[1].components[0] osgi1-instance1 +chains[1].components[1] osgi1-instance2 +chains[1].components[2] osgi2-instance2 +chains[2].id multiOsgiInstances +chains[2].components[4] +chains[2].components[0] osgim1-instance1 +chains[2].components[1] osgim1-instance2 +chains[2].components[2] osgi2-instance2 +chains[2].components[3] osgim2-instance2 +components[9] +components[0].id class1-instance1 +components[1].id class1-instance2 +components[2].id class2-instance2 +components[3].id osgi1-instance1 +components[4].id osgi1-instance2 +components[5].id osgi2-instance2 +components[6].id osgim1-instance1 +components[7].id osgim1-instance2 +components[8].id osgim2-instance2 diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg new file mode 100644 index 00000000000..8a985f92d10 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/components.cfg @@ -0,0 +1,24 @@ +components[11] +components[0].id class1-instance1 +components[0].classId com.yahoo.search.searchchain.config.test.SearcherInstancesTestCase$Searcher1 +components[1].id class1-instance2 +components[1].classId com.yahoo.search.searchchain.config.test.SearcherInstancesTestCase$Searcher1 +components[2].id class2-instance2 +components[2].classId com.yahoo.search.searchchain.config.test.SearcherInstancesTestCase$Searcher2 +components[3].id osgi1-instance1 +components[3].classId com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +components[4].id osgi1-instance2 +components[4].classId com.yahoo.search.searchchain.config.test.searcher1.Searcher1 +components[5].id osgi2-instance2 +components[5].classId com.yahoo.search.searchchain.config.test.searcher2.Searcher2 +components[6].id osgim1-instance1 +components[6].classId com.yahoo.search.searchchain.config.test.twosearchers.MultiSearcher1 +components[6].bundle twosearchers +components[7].id osgim1-instance2 +components[7].classId com.yahoo.search.searchchain.config.test.twosearchers.MultiSearcher1 +components[7].bundle twosearchers +components[8].id osgim2-instance2 +components[8].classId com.yahoo.search.searchchain.config.test.twosearchers.MultiSearcher2 +components[8].bundle twosearchers +components[9].id com.yahoo.search.handler.SearchHandler +components[10].id com.yahoo.container.handler.config.HandlersConfigurerDi$RegistriesHack diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/handlers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/handlers.cfg new file mode 100644 index 00000000000..ad20005e7ad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/handlers.cfg @@ -0,0 +1,2 @@ +handler[1] +handler[0].id com.yahoo.search.handler.SearchHandler diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/qr-searchers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/qr-searchers.cfg new file mode 100644 index 00000000000..949eae83da5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/qr-searchers.cfg @@ -0,0 +1,4 @@ + +customizedsearchers.transformedquery[0] + +customizedsearchers.argument[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/specialtokens.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/specialtokens.cfg new file mode 100644 index 00000000000..5b5b5ab6a15 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/specialtokens.cfg @@ -0,0 +1 @@ +tokenlist[0] diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher.cfg new file mode 100644 index 00000000000..712b7071447 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher.cfg @@ -0,0 +1,6 @@ +chains[1] +chains[0].id update-searcher +chains[0].components[1] +chains[0].components[0] update-searcher +components[1] +components[0].id update-searcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher2.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher2.cfg new file mode 100644 index 00000000000..39b0237deb0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/testInstances/updatesearcher2.cfg @@ -0,0 +1,7 @@ +chains[1] +chains[0].id update-searcher +chains[0].components[1] +chains[0].components[0] update-searcher +components[2] +components[0].id update-searcher +components[1].id update-searcher2 diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/three-searchers.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/three-searchers.cfg new file mode 100644 index 00000000000..13ec94e9aa5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/three-searchers.cfg @@ -0,0 +1,10 @@ +chains[1] +chains[0].id three-searchers +chains[0].components[3] +chains[0].components[0] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +chains[0].components[1] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$StringSearcher +chains[0].components[2] com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher +components[3] +components[0].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$IntSearcher +components[1].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$StringSearcher +components[2].id com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase$DeclaredTestSearcher diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/Manifest.MF new file mode 100644 index 00000000000..20260ba2733 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/Manifest.MF @@ -0,0 +1,11 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: HelloWorld +Bundle-SymbolicName: twosearchers +Bundle-Version: 1.0.0 +Bundle-Vendor: Yahoo! +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher1.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher1.java new file mode 100644 index 00000000000..0e7e5fb54b7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher1.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.twosearchers; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class MultiSearcher1 extends Searcher { + + public Result search(Query query, Execution execution) { + return execution.search(query); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher2.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher2.java new file mode 100644 index 00000000000..091380a524e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/twosearchers/MultiSearcher2.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.twosearchers; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class MultiSearcher2 extends Searcher { + + public Result search(Query query, Execution execution) { + return execution.search(query); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/Manifest.MF new file mode 100644 index 00000000000..7c98e11231e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.updatesearcher.UpdateSearcher +Bundle-Version: 0 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/UpdateSearcher.java.text b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/UpdateSearcher.java.text new file mode 100644 index 00000000000..f62333761a2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher-2/UpdateSearcher.java.text @@ -0,0 +1,24 @@ +package com.yahoo.search.searchchain.config.test.updatesearcher; + +import com.yahoo.search.Searcher; +import com.yahoo.search.Result; +import com.yahoo.search.Query; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class UpdateSearcher extends com.yahoo.search.Searcher { + + public String test = "update2"; + + public @Override + Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:updatesearcher:2")); + return result; + } +}
\ No newline at end of file diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/Manifest.MF b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/Manifest.MF new file mode 100644 index 00000000000..7c98e11231e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/Manifest.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Searcher1 +Bundle-SymbolicName: com.yahoo.search.searchchain.config.test.updatesearcher.UpdateSearcher +Bundle-Version: 0 +Bundle-Vendor: Yahoo! +Export-Package: com.yahoo.search.searchchain.config.test.searcher1 +Import-Package: org.osgi.framework;version="1.3.0", + com.yahoo.component, + com.yahoo.search.result, + com.yahoo.search.searchchain, + com.yahoo.search diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java new file mode 100644 index 00000000000..6c38df8fe08 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/config/test/updatesearcher/UpdateSearcher.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.config.test.updatesearcher; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author bratseth + */ +public class UpdateSearcher extends com.yahoo.search.Searcher { + + public String test = "update"; + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + if (result==null) + result=new Result(query); + result.hits().add(new Hit("from:updatesearcher:0")); + return result; + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/model/test/chains.cfg b/container-search/src/test/java/com/yahoo/search/searchchain/model/test/chains.cfg new file mode 100644 index 00000000000..3c19c691e9b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/model/test/chains.cfg @@ -0,0 +1,33 @@ +chains[1] +chains[0].id "default_chain" +chains[0].configId searchchains/searchchain/default_chain +chains[0].components[3] +chains[0].components[0] InitSearcher +chains[0].components[1] PrepareSearcher +chains[0].components[2] RunSearcher +chains[0].phases[1] +chains[0].phases[0].id "phase_1" +chains[0].phases[0].before[1] +chains[0].phases[0].before[0] phase_2 +components[3] +components[0].id "InitSearcher" +components[0].configId searchchains/searcher/InitSearcher +components[0].dependencies.before[1] +components[0].dependencies.before[0] init +components[0].dependencies.after[1] +components[0].dependencies.after[0] prepare +components[1].id "PrepareSearcher" +components[1].classid "PrepareSearcherClass" +components[1].configId searchchains/searcher/PrepareSearcher +components[1].dependencies.provides[1] +components[1].dependencies.provides[0] init +components[1].dependencies.before[1] +components[1].dependencies.before[0] prepare +components[2].id "RunSearcher" +components[2].classid "RunSearcherClass" +components[2].bundle "RunSearcherBundle" +components[2].configId searchchains/searcher/RunSearcher +components[2].dependencies.provides[1] +components[2].dependencies.provides[0] prepare +components[2].dependencies.before[1] +components[2].dependencies.before[0] run diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionOfOneChainTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionOfOneChainTestCase.java new file mode 100644 index 00000000000..482b3e08661 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionOfOneChainTestCase.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.AsyncExecution; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.FutureResult; +import junit.framework.TestCase; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class AsyncExecutionOfOneChainTestCase extends TestCase { + + /** Tests having a result with some slow source data which should pass directly to rendering */ + public void testParallelExecutionOfOneChain() { + // Setup + Chain<Searcher> mainChain=new Chain<>(new ParallelExecutor(),new ResultProcessor(),new RegularProvider()); + + // Execute + Result result=new Execution(mainChain, Execution.Context.createContextStub()).search(new Query()); + + // Verify + assertEquals("Received 2 hits from 3 threads",3*2,result.hits().size()); + assertEquals(1.0, result.hits().get("thread-0:hit-0").getRelevance().getScore()); + assertEquals(1.0, result.hits().get("thread-1:hit-0").getRelevance().getScore()); + assertEquals(1.0, result.hits().get("thread-2:hit-0").getRelevance().getScore()); + assertEquals(0.5, result.hits().get("thread-0:hit-1").getRelevance().getScore()); + assertEquals(0.5, result.hits().get("thread-1:hit-1").getRelevance().getScore()); + assertEquals(0.5, result.hits().get("thread-2:hit-1").getRelevance().getScore()); + } + + private class ParallelExecutor extends Searcher { + + /** The number of parallel executions */ + private static final int parallelism=2; + + @Override + public Result search(Query query, Execution execution) { + List<FutureResult> futureResults=new ArrayList<>(parallelism); + for (int i=0; i<parallelism; i++) + futureResults.add(new AsyncExecution(execution).search(query.clone())); + + Result mainResult=execution.search(query); + + // Add hits from other threads + AsyncExecution.waitForAll(futureResults,query.getTimeLeft()); + for (FutureResult futureResult : futureResults) { + Result result=futureResult.get(); + mainResult.mergeWith(result); + mainResult.hits().addAll(result.hits().asList()); + } + return mainResult; + } + + } + + private static class RegularProvider extends Searcher { + + private AtomicInteger counter=new AtomicInteger(); + + @Override + public Result search(Query query,Execution execution) { + String thread="thread-" + counter.getAndIncrement(); + Result result=new Result(query,new HitGroup("test")); + result.hits().add(new Hit(thread + ":hit-0",1.0)); + result.hits().add(new Hit(thread + ":hit-1",0.9)); + return result; + } + + } + + private static class ResultProcessor extends Searcher { + + @Override + public Result search(Query query,Execution execution) { + Result result=execution.search(query); + + int i=1; + for (Iterator<Hit> hits=result.hits().deepIterator(); hits.hasNext(); ) + hits.next().setRelevance(1d/i++); + return result; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionTestCase.java new file mode 100644 index 00000000000..a54367628a3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/AsyncExecutionTestCase.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.AsyncExecution; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.FutureResult; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Test for aynchrounous execution + * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + */ +public class AsyncExecutionTestCase extends junit.framework.TestCase { + + public class WaitingSearcher extends Searcher { + + int waittime; + private WaitingSearcher(String id,int waittime) { + super(new ComponentId(id)); + this.waittime = waittime; + } + + @Override + public Result search(Query query,Execution execution) { + Result result=execution.search(query); + if(waittime != 0) + try { + Thread.sleep(waittime); + } catch (InterruptedException e) { + } + return result; + } + } + + public class SimpleSearcher extends Searcher { + + public Result search(Query query,Execution execution) { + return execution.search(query); + } + + } + + //This should take ~50+ ms + public void testAsync() { + List<Searcher> searchList = new ArrayList<>(); + searchList.add(new WaitingSearcher("one",60000)); + searchList.add(new WaitingSearcher("two",0)); + Chain<Searcher> searchChain = new Chain<>(new ComponentId("chain"), searchList); + + AsyncExecution asyncExecution = new AsyncExecution(searchChain, Execution.Context.createContextStub()); + FutureResult future = asyncExecution.search(new Query("?hits=0")); + Result result = future.get(0, TimeUnit.MILLISECONDS); + + assertTrue(result.hits().getError() != null); + } + + public void testWaitForAll() { + Chain<Searcher> slowChain = new Chain<>( + new ComponentId("slow"), + Arrays.asList(new Searcher[]{new WaitingSearcher("slow",30000)} + ) + ); + + Chain<Searcher> fastChain = new Chain<>( + new ComponentId("fast"), + Arrays.asList(new Searcher[]{new SimpleSearcher()}) + ); + + FutureResult slowFuture = new AsyncExecution(slowChain, Execution.Context.createContextStub()).search(new Query("?hits=0")); + FutureResult fastFuture = new AsyncExecution(fastChain, Execution.Context.createContextStub()).search(new Query("?hits=0")); + fastFuture.get(); + FutureResult reslist[] = new FutureResult[]{slowFuture,fastFuture}; + List<Result> results = AsyncExecution.waitForAll(Arrays.asList(reslist),0); + + //assertTrue(slowFuture.isCancelled()); + assertTrue(fastFuture.isDone() && !fastFuture.isCancelled()); + + assertNotNull(results.get(0).hits().getErrorHit()); + assertNull(results.get(1).hits().getErrorHit()); + } + + public void testSync() { + Query query=new Query("?query=test"); + Searcher searcher=new ResultProducingSearcher(); + Result result=new Execution(searcher, Execution.Context.createContextStub()).search(query); + + assertEquals(1,result.hits().size()); + assertEquals("hello",result.hits().get(0).getField("test")); + } + + public void testSyncThroughSync() { + Query query=new Query("?query=test"); + Searcher searcher=new ResultProducingSearcher(); + Result result=new Execution(new Execution(searcher, Execution.Context.createContextStub())).search(query); + + assertEquals(1,result.hits().size()); + assertEquals("hello",result.hits().get(0).getField("test")); + } + + public void testAsyncThroughSync() { + Query query=new Query("?query=test"); + Searcher searcher=new ResultProducingSearcher(); + FutureResult futureResult=new AsyncExecution(new Execution(searcher, Execution.Context.createContextStub())).search(query); + + List<FutureResult> futureResultList=new ArrayList<>(); + futureResultList.add(futureResult); + AsyncExecution.waitForAll(futureResultList,1000); + Result result=futureResult.get(); + + assertEquals(1,result.hits().size()); + assertEquals("hello",result.hits().get(0).getField("test")); + } + + private static class ResultProducingSearcher extends Searcher { + + @Override + public Result search(Query query,Execution execution) { + Result result=new Result(query); + Hit hit=new Hit("test"); + hit.setField("test","hello"); + result.hits().add(hit); + return result; + } + + } + + @SuppressWarnings("deprecation") + public void testAsyncExecutionTimeout() { + Chain<Searcher> chain = new Chain<>(new Searcher() { + @Override + public Result search(Query query, Execution execution) { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return new Result(query); + } + }); + Execution execution = new Execution(chain, Execution.Context.createContextStub()); + AsyncExecution async = new AsyncExecution(execution); + FutureResult future = async.searchAndFill(new Query()); + future.get(1, TimeUnit.MILLISECONDS); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java new file mode 100644 index 00000000000..642d8d8cd7e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/ExecutionTestCase.java @@ -0,0 +1,297 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import org.junit.Test; + +/** + * Tests basic search chain execution functionality + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class ExecutionTestCase extends junit.framework.TestCase { + + public void testLinearExecutions() { + // Make a chain + List<Searcher> searchers1=new ArrayList<>(); + searchers1.add(new TestSearcher("searcher1")); + searchers1.add(new TestSearcher("searcher2")); + searchers1.add(new TestSearcher("searcher3")); + searchers1.add(new TestSearcher("searcher4")); + Chain<Searcher> chain1=new Chain<>(new ComponentId("chain1"), searchers1); + // Make another chain containing two of the same searcher instances and two new + List<Searcher> searchers2=new ArrayList<>(searchers1); + searchers2.set(1,new TestSearcher("searcher5")); + searchers2.set(3,new TestSearcher("searcher6")); + Chain<Searcher> chain2=new Chain<>(new ComponentId("chain2"), searchers2); + // Execute both + Query query=new Query("test"); + Result result1=new Execution(chain1, Execution.Context.createContextStub()).search(query); + Result result2=new Execution(chain2, Execution.Context.createContextStub()).search(query); + // Verify results + assertEquals(4,result1.getConcreteHitCount()); + assertNotNull(result1.hits().get("searcher1-1")); + assertNotNull(result1.hits().get("searcher2-1")); + assertNotNull(result1.hits().get("searcher3-1")); + assertNotNull(result1.hits().get("searcher4-1")); + + assertEquals(4,result2.getConcreteHitCount()); + assertNotNull(result2.hits().get("searcher1-2")); + assertNotNull(result2.hits().get("searcher5-1")); + assertNotNull(result2.hits().get("searcher3-2")); + assertNotNull(result2.hits().get("searcher6-1")); + } + + public void testNestedExecution() { + // Make a chain + List<Searcher> searchers1=new ArrayList<>(); + searchers1.add(new FillableTestSearcher("searcher1")); + searchers1.add(new WorkflowSearcher()); + searchers1.add(new TestSearcher("searcher2")); + searchers1.add(new FillingSearcher()); + searchers1.add(new FillableTestSearcherAtTheEnd("searcher3")); + Chain<Searcher> chain1=new Chain<>(new ComponentId("chain1"), searchers1); + // Execute it + Query query=new Query("test"); + Result result1=new Execution(chain1, Execution.Context.createContextStub()).search(query); + // Verify results + assertEquals(7,result1.getConcreteHitCount()); + assertNotNull(result1.hits().get("searcher1-1")); + assertNotNull(result1.hits().get("searcher2-1")); + assertNotNull(result1.hits().get("searcher3-1")); + assertNotNull(result1.hits().get("searcher3-1-filled")); + assertNotNull(result1.hits().get("searcher2-2")); + assertNotNull(result1.hits().get("searcher3-2")); + assertNotNull(result1.hits().get("searcher3-2-filled")); + } + + public void testContextCacheSingleLengthSearchChain() { + IndexFacts[] contextsBefore = new IndexFacts[1]; + IndexFacts[] contextsAfter = new IndexFacts[1]; + List<Searcher> l = new ArrayList<>(1); + l.add(new ContextCacheSearcher(0, contextsBefore, contextsAfter)); + Chain<Searcher> chain = new Chain<>(l); + Query query = new Query("?mutatecontext=0"); + new Execution(chain, Execution.Context.createContextStub()).search(query); + assertEquals(contextsBefore[0], contextsAfter[0]); + assertSame(contextsBefore[0], contextsAfter[0]); + } + + public void testContextCache() { + IndexFacts[] contextsBefore = new IndexFacts[5]; + IndexFacts[] contextsAfter = new IndexFacts[5]; + List<Searcher> l = new ArrayList<>(5); + l.add(new ContextCacheSearcher(0, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(1, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(2, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(3, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(4, contextsBefore, contextsAfter)); + Chain<Searcher> chain = new Chain<>(l); + Query query = new Query("?mutatecontext=2"); + new Execution(chain, Execution.Context.createContextStub()).search(query); + + assertSame(contextsBefore[0], contextsAfter[0]); + assertSame(contextsBefore[1], contextsAfter[1]); + assertSame(contextsBefore[2], contextsAfter[2]); + assertSame(contextsBefore[3], contextsAfter[3]); + assertSame(contextsBefore[4], contextsAfter[4]); + + assertSame(contextsBefore[0], contextsBefore[1]); + assertNotSame(contextsBefore[1], contextsBefore[2]); + assertSame(contextsBefore[2], contextsBefore[3]); + assertSame(contextsBefore[3], contextsBefore[4]); + } + + public void testContextCacheMoreSearchers() { + IndexFacts[] contextsBefore = new IndexFacts[7]; + IndexFacts[] contextsAfter = new IndexFacts[7]; + List<Searcher> l = new ArrayList<>(7); + l.add(new ContextCacheSearcher(0, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(1, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(2, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(3, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(4, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(5, contextsBefore, contextsAfter)); + l.add(new ContextCacheSearcher(6, contextsBefore, contextsAfter)); + Chain<Searcher> chain = new Chain<>(l); + Query query = new Query("?mutatecontext=2,4"); + new Execution(chain, Execution.Context.createContextStub()).search(query); + + assertSame(contextsBefore[0], contextsAfter[0]); + assertSame(contextsBefore[1], contextsAfter[1]); + assertSame(contextsBefore[2], contextsAfter[2]); + assertSame(contextsBefore[3], contextsAfter[3]); + assertSame(contextsBefore[4], contextsAfter[4]); + assertSame(contextsBefore[5], contextsAfter[5]); + assertSame(contextsBefore[6], contextsAfter[6]); + + assertSame(contextsBefore[0], contextsBefore[1]); + assertNotSame(contextsBefore[1], contextsBefore[2]); + assertSame(contextsBefore[2], contextsBefore[3]); + assertNotSame(contextsBefore[3], contextsBefore[4]); + assertSame(contextsBefore[4], contextsBefore[5]); + assertSame(contextsBefore[5], contextsBefore[6]); + } + + @Test + public void testBasicFill() { + Chain<Searcher> chain = new Chain<Searcher>(new FillableResultSearcher()); + Execution execution = new Execution(chain, Execution.Context.createContextStub(null)); + + Result result = execution.search(new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?presentation.summary=all"))); + assertNotNull(result.hits().get("a")); + assertNull(result.hits().get("a").getField("filled")); + execution.fill(result); + assertTrue((Boolean) result.hits().get("a").getField("filled")); + } + + private static class FillableResultSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + Hit hit = new Hit("a"); + hit.setFillable(); + result.hits().add(hit); + return result; + } + + @Override + public void fill(Result result, String summaryClass, Execution execution) { + for (Hit hit : result.hits().asList()) { + if ( ! hit.isFillable()) continue; + hit.setField("filled",true); + hit.setFilled("all"); + } + } + } + + static class ContextCacheSearcher extends Searcher { + final int index; + final IndexFacts[] contextsBefore; + final IndexFacts[] contextsAfter; + + ContextCacheSearcher(int index, IndexFacts[] contextsBefore, IndexFacts[] contextsAfter) { + this.index = index; + this.contextsBefore = contextsBefore; + this.contextsAfter = contextsAfter; + } + + @Override + public Result search(Query query, Execution execution) { + String s = query.properties().getString("mutatecontext"); + Set<Integer> indexSet = new HashSet<>(); + for (String num : s.split(",")) { + indexSet.add(Integer.valueOf(num)); + } + + if (indexSet.contains(index)) { + execution.context().setIndexFacts(new IndexFacts()); + } + contextsBefore[index] = execution.context().getIndexFacts(); + Result r = execution.search(query); + contextsAfter[index] = execution.context().getIndexFacts(); + return r; + } + } + + public static class TestSearcher extends Searcher { + + private int counter=1; + + private TestSearcher(String id) { + super(new ComponentId(id)); + } + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + result.hits().add(new Hit(getId().stringValue() + "-" + (counter++))); + return result; + } + + } + + public static class ForwardingSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Chain<Searcher> forwardTo=execution.context().searchChainRegistry().getChain("someChainId"); + return new Execution(forwardTo,execution.context()).search(query); + + } + + } + + public static class FillableTestSearcher extends Searcher { + + private int counter=1; + + private FillableTestSearcher(String id) { + super(new ComponentId(id)); + } + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + Hit hit=new Hit(getId().stringValue() + "-" + counter); + hit.setFillable(); + result.hits().add(hit); + return result; + } + + public @Override void fill(Result result,String summaryClass,Execution execution) { + result.hits().add(new Hit(getId().stringValue() + "-" + (counter++) + "-filled")); // Not something one would normally do in fill + } + + } + + public static class FillableTestSearcherAtTheEnd extends FillableTestSearcher { + + private FillableTestSearcherAtTheEnd(String id) { + super(id); + } + } + + @Before("com.yahoo.search.searchchain.test.ExecutionTestCase$FillableTestSearcherAtTheEnd") + @After("com.yahoo.search.searchchain.test.ExecutionTestCase$TestSearcher") + public static class FillingSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result=execution.search(query); + execution.fill(result); + return result; + } + + } + + @After("com.yahoo.search.searchchain.test.ExecutionTestCase$FillableTestSearcher") + @Before("com.yahoo.search.searchchain.test.ExecutionTestCase$TestSearcher") + public static class WorkflowSearcher extends Searcher { + + public @Override Result search(Query query,Execution execution) { + Result result1=execution.search(query); + Result result2=execution.search(query); + for (Iterator<Hit> i=result2.hits().iterator(); i.hasNext();) + result1.hits().add(i.next()); + result1.mergeWith(result2); + return result1; + } + + } + + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java new file mode 100644 index 00000000000..79881c06852 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/FutureDataTestCase.java @@ -0,0 +1,150 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.google.common.util.concurrent.AbstractFuture; +import com.google.common.util.concurrent.ListenableFuture; +import com.yahoo.component.ComponentId; +import com.yahoo.processing.response.*; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.federation.FederationSearcher; +import com.yahoo.search.federation.sourceref.SearchChainResolver; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.search.searchchain.Execution; + +import com.yahoo.search.searchchain.SearchChainRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.junit.Test; +import static org.junit.Assert.*; + +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.yahoo.component.chain.Chain; + +/** + * Tests using the async capabilities of the Processing parent framework of searchers. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class FutureDataTestCase { + + @Test + public void testAsyncFederation() throws InterruptedException, ExecutionException, TimeoutException { + // Setup environment + AsyncProviderSearcher asyncProviderSearcher = new AsyncProviderSearcher(); + Searcher syncProviderSearcher = new SyncProviderSearcher(); + Chain<Searcher> asyncSource = new Chain<Searcher>(new ComponentId("async"),asyncProviderSearcher); + Chain<Searcher> syncSource = new Chain<>(new ComponentId("sync"),syncProviderSearcher); + SearchChainResolver searchChainResolver= + new SearchChainResolver.Builder().addSearchChain(new ComponentId("sync"),new FederationOptions().setUseByDefault(true)). + addSearchChain(new ComponentId("async"),new FederationOptions().setUseByDefault(true)). + build(); + Chain<Searcher> main = new Chain<Searcher>(new FederationSearcher(new ComponentId("federator"),searchChainResolver)); + SearchChainRegistry searchChainRegistry = new SearchChainRegistry(); + searchChainRegistry.register(main); + searchChainRegistry.register(syncSource); + searchChainRegistry.register(asyncSource); + + Result result = new Execution(main, Execution.Context.createContextStub(searchChainRegistry,null)).search(new Query()); + assertNotNull(result); + + HitGroup syncGroup = (HitGroup)result.hits().get("source:sync"); + assertNotNull(syncGroup); + + HitGroup asyncGroup = (HitGroup)result.hits().get("source:async"); + assertNotNull(asyncGroup); + + assertEquals("Got all sync data",3,syncGroup.size()); + assertEquals("sync:0",syncGroup.get(0).getId().toString()); + assertEquals("sync:1",syncGroup.get(1).getId().toString()); + assertEquals("sync:2",syncGroup.get(2).getId().toString()); + + assertTrue(asyncGroup.incoming()==asyncProviderSearcher.incomingData); + assertEquals("Got no async data yet",0,asyncGroup.size()); + asyncProviderSearcher.simulateOneHitIOComplete(new Hit("async:0")); + assertEquals("Got no async data yet, as we haven't completed the incoming buffer and there is no data listener",0,asyncGroup.size()); + asyncProviderSearcher.simulateOneHitIOComplete(new Hit("async:1")); + asyncProviderSearcher.simulateAllHitsIOComplete(); + assertEquals("Got no async data yet, as we haven't pulled it",0,asyncGroup.size()); + asyncGroup.complete().get(); + assertEquals("Completed, so we have the data",2,asyncGroup.size()); + assertEquals("async:0",asyncGroup.get(0).getId().toString()); + assertEquals("async:1",asyncGroup.get(1).getId().toString()); + } + + @Test + public void testFutureData() throws InterruptedException, ExecutionException, TimeoutException { + // Set up + AsyncProviderSearcher futureDataSource=new AsyncProviderSearcher(); + Chain<Searcher> chain=new Chain<>(Collections.<Searcher>singletonList(futureDataSource)); + + // Execute + Query query = new Query(); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + + // Verify the result prior to completion of delayed data + assertEquals("The result has been returned, but no hits are available yet", + 0, result.hits().getConcreteSize()); + + // pretend we're the IO layer and complete delayed data - this is typically done in a callback from jDisc + futureDataSource.simulateOneHitIOComplete(new Hit("hit:0")); + futureDataSource.simulateOneHitIOComplete(new Hit("hit:1")); + futureDataSource.simulateAllHitsIOComplete(); + + assertEquals("Async arriving hits are still not visible because we haven't asked for them", + 0, result.hits().getConcreteSize()); + + // Results with future hit groups will be passed to rendering directly and start rendering immediately. + // For this test we block and wait for the data instead: + result.hits().complete().get(1000, TimeUnit.MILLISECONDS); + assertEquals(2,result.hits().getConcreteSize()); + } + + /** + * A searcher which returns immediately with future data which can then be filled later, + * simulating an async searcher using a separate thread to fill in result data as it becomes available. + */ + public static class AsyncProviderSearcher extends Searcher { + + private IncomingData<Hit> incomingData = null; + + @Override + public Result search(Query query, Execution execution) { + if (incomingData != null) throw new IllegalArgumentException("This test searcher is one-time use only"); + + HitGroup hitGroup=HitGroup.createAsync("Async source"); + this.incomingData = hitGroup.incoming(); + // A real implementation would do query.properties().get("jdisc.request") here + // to get the jDisc request and use it to spawn a child request to the backend + // which would eventually add to and complete incomingData + return new Result(query,hitGroup); + } + + public void simulateOneHitIOComplete(Hit hit) { + incomingData.add(hit); + } + + public void simulateAllHitsIOComplete() { + incomingData.markComplete(); + } + + } + + public static class SyncProviderSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + result.hits().add(new Hit("sync:0")); + result.hits().add(new Hit("sync:1")); + result.hits().add(new Hit("sync:2")); + return result; + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java new file mode 100644 index 00000000000..ad0c4796549 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/SearchChainTestCase.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import static com.yahoo.search.searchchain.test.SimpleSearchChain.searchChain; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.Version; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.SearchChain; + +/** + * Tests basic search chain functionality - creation, inheritance and ordering + * + * @author bratseth + */ +@SuppressWarnings("deprecation") +public class SearchChainTestCase extends junit.framework.TestCase { + + public SearchChainTestCase(String name) { + super(name); + } + + public void testEmptySearchChain() { + SearchChain empty = new SearchChain(new ComponentId("empty")); + assertEquals("empty", empty.getId().getName()); + } + + public void testSearchChainCreation() { + assertEquals("test",searchChain.getId().stringValue()); + assertEquals("test",searchChain.getId().getName()); + assertEquals(Version.emptyVersion, searchChain.getId().getVersion()); + assertEquals(new Version(),searchChain.getId().getVersion()); + assertEqualMembers(Arrays.asList("one", "two"), searcherNames(searchChain.searchers())); + } + + public List<String> searcherNames(Collection<Searcher> searchers) { + List<String> names = new ArrayList<>(); + + for (Searcher searcher: searchers) { + names.add(searcher.getId().stringValue()); + } + + Collections.sort(names); + return names; + } + + private void assertEqualMembers(List<String> correct,List<?> test) { + assertEquals(new HashSet<>(correct),new HashSet<>(test)); + } + + public void testSearchChainToStringEmpty() { + assertEquals("chain 'test' []", new Chain<>(new ComponentId("test"), createSearchers(0)).toString()); + } + + public void testSearchChainToStringVeryShort() { + assertEquals("chain 'test' [s1]", new Chain<>(new ComponentId("test"),createSearchers(1)).toString()); + } + + public void testSearchChainToStringShort() { + assertEquals("chain 'test' [s1 -> s2 -> s3]", new Chain<>(new ComponentId("test"),createSearchers(3)).toString()); + } + + public void testSearchChainToStringLong() { + assertEquals("chain 'test' [s1 -> s2 -> ... -> s4]", new Chain<>(new ComponentId("test"),createSearchers(4)).toString()); + } + + public void testSearchChainToStringVeryLong() { + assertEquals("chain 'test' [s1 -> s2 -> ... -> s10]", new Chain<>(new ComponentId("test"),createSearchers(10)).toString()); + } + + private List<Searcher> createSearchers(int count) { + List<Searcher> searchers=new ArrayList<>(count); + for (int i=0; i<count; i++) + searchers.add(new TestSearcher("s" + String.valueOf(i+1))); + return searchers; + } + + private static class TestSearcher extends Searcher { + + private TestSearcher(String id) { + super(new ComponentId(id)); + } + + public Result search(Query query, Execution execution) { + return execution.search(query); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/SimpleSearchChain.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/SimpleSearchChain.java new file mode 100644 index 00000000000..8aeef025271 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/SimpleSearchChain.java @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.dependencies.After; +import com.yahoo.component.chain.dependencies.Provides; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.ForkingSearcher; +import com.yahoo.search.searchchain.SearchChain; +import com.yahoo.search.searchchain.SearchChainRegistry; + +/** + * A search chain consisting of two searchers. + * @author bratseth + * @author tonytv + */ +public class SimpleSearchChain { + + private static abstract class BaseSearcher extends ForkingSearcher { + + public BaseSearcher(ComponentId id) { + super(); + initId(id); + } + + @Override + public Result search(Query query,Execution execution) { + return execution.search(query); + } + + @Override + public Collection<ForkingSearcher.CommentedSearchChain> getSearchChainsForwarded(SearchChainRegistry registry) { + return Arrays.asList( + new ForkingSearcher.CommentedSearchChain("Reason for forwarding to this search chain.", dummySearchChain()), + new ForkingSearcher.CommentedSearchChain(null, dummySearchChain())); + } + + private SearchChain dummySearchChain() { + return new SearchChain(new ComponentId("child-chain"), + new DummySearcher(new ComponentId("child-searcher")) {}); + } + + } + + @Provides("Test") + private static class TestSearcher extends BaseSearcher { + + public TestSearcher(ComponentId id) { + super(id); + } + + } + + private static class DummySearcher extends Searcher { + + public DummySearcher(ComponentId id) { + super(id); + } + + @Override + public Result search(Query query,Execution execution) { + return execution.search(query); + } + + } + + @After("Test") + private static class TestSearcher2 extends BaseSearcher { + + public TestSearcher2(ComponentId id) { + super(id); + } + + @Override + public Result search(Query query,Execution execution) { + return execution.search(query); + } + + } + + private static List<Searcher> twoSearchers(String id1, String id2, boolean ordered) { + List<Searcher> searchers=new ArrayList<>(); + searchers.add(new TestSearcher(new ComponentId(id1))); + searchers.add(createSecondSearcher(new ComponentId(id2), ordered)); + return searchers; + } + + private static Searcher createSecondSearcher(ComponentId componentId, boolean ordered) { + if (ordered) + return new TestSearcher2(componentId); + else + return new TestSearcher(componentId); + } + + private static SearchChain createSearchChain(boolean ordered) { + return new SearchChain(new ComponentId("test"), twoSearchers("one","two", ordered)); + } + + public static final SearchChain searchChain = createSearchChain(false); + public static final SearchChain orderedChain = createSearchChain(true); + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java new file mode 100644 index 00000000000..92091fabbf9 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/TraceTestCase.java @@ -0,0 +1,221 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.yolean.trace.TraceNode; +import com.yahoo.yolean.trace.TraceVisitor; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +/** + * Tests tracing scenarios where traces from multiple executions over the same query are involved. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class TraceTestCase extends junit.framework.TestCase { + + public void testTracingOnCorrectAPIUseNonParallel() { + assertTracing(true,false); + } + + public void testTracingOnIncorrectAPIUseNonParallel() { + assertTracing(false,false); + } + + public void testTracingOnCorrectAPIUseParallel() { + assertTracing(true, true); + } + + public void testTracingOnIncorrectAPIUseParallel() { + assertTracing(false,true); + } + + @SuppressWarnings("deprecation") + public void assertTracing(boolean carryOverContext,boolean parallel) { + Query query=new Query("?tracelevel=1"); + query.trace("Before execution",1); + Chain<Searcher> forkingChain=new Chain<>(new Tracer("forker"),new Forker(carryOverContext,parallel,new Tracer("branch 1"),new Tracer("branch 2"))); + new Execution(forkingChain, Execution.Context.createContextStub()).search(query); + + // printTrace(query); + + if (carryOverContext) + assertTraceWithChildExecutionMessages(query); + else if (parallel) + assertTrace(query); + else + assertIncorrectlyNestedTrace(query); + + assertCorrectRendering(query); + } + + // The valid and usual trace + private void assertTraceWithChildExecutionMessages(Query query) { + Iterator<String> trace=collectTrace(query).iterator(); + assertEquals("(level start)",trace.next()); + assertEquals(" No query profile is used",trace.next()); + assertEquals(" Before execution",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During forker: 0",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 1: 0",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 2: 0", trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals("(level end)",trace.next()); + assertFalse(trace.hasNext()); + } + + // With incorrect API usage and query cloning (in parallel use) we get a valid trace + // where the message of the execution subtrees is empty rather than "child execution". This is fine. + private void assertTrace(Query query) { + Iterator<String> trace=collectTrace(query).iterator(); + assertEquals("(level start)",trace.next()); + assertEquals(" No query profile is used",trace.next()); + assertEquals(" Before execution",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During forker: 0",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 1: 0",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 2: 0", trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals("(level end)",trace.next()); + assertFalse(trace.hasNext()); + } + + // With incorrect usage and no query cloning the trace nesting becomes incorrect + // but all the trace messages are present. + private void assertIncorrectlyNestedTrace(Query query) { + Iterator<String> trace=collectTrace(query).iterator(); + assertEquals("(level start)",trace.next()); + assertEquals(" No query profile is used",trace.next()); + assertEquals(" Before execution",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During forker: 0",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 1: 0",trace.next()); + assertEquals(" (level start)",trace.next()); + assertEquals(" During branch 2: 0", trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals(" (level end)",trace.next()); + assertEquals("(level end)",trace.next()); + assertFalse(trace.hasNext()); + } + + private void assertCorrectRendering(Query query) { + try { + StringWriter writer=new StringWriter(); + query.getContext(false).render(writer); + String expected= + "<meta type=\"context\">\n" + + "\n" + + " <p>No query profile is used</p>\n" + + "\n" + + " <p>Before execution</p>\n" + + "\n" + + " <p>\n" + + " <p>During forker: 0"; + assertEquals(expected,writer.toString().substring(0,expected.length())); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + private List<String> collectTrace(Query query) { + TraceCollector collector=new TraceCollector(); + query.getContext(false).getTrace().accept(collector); + return collector.trace(); + } + + private static class TraceCollector extends TraceVisitor { + + private List<String> trace=new ArrayList<>(); + private StringBuilder indent=new StringBuilder(); + + @Override + public void entering(TraceNode node) { + trace.add(indent + "(level start)"); + indent.append(" "); + } + + @Override + public void leaving(TraceNode end) { + indent.setLength(indent.length()-2); + trace.add(indent + "(level end)"); + } + + @Override + public void visit(TraceNode node) { + if (node.isRoot()) return; + if (node.payload()==null) return; + trace.add(indent + node.payload().toString()); + } + + public List<String> trace() { return trace; } + } + + private static class Tracer extends Searcher { + + private String name; + private int counter=0; + + public Tracer(String name) { + this.name=name; + } + + @Override + public Result search(Query query, Execution execution) { + query.trace("During " + name + ": " + (counter++) ,1); + return execution.search(query); + } + } + + private static class Forker extends Searcher { + + private List<Searcher> branches; + + /** If true, this is using the api as recommended, if false, it is not */ + private boolean carryOverContext; + + /** If true, simulate parallel execution by cloning the query */ + private boolean parallel; + + public Forker(boolean carryOverContext,boolean parallel,Searcher ... branches) { + this.carryOverContext=carryOverContext; + this.parallel=parallel; + this.branches=Arrays.asList(branches); + } + + @SuppressWarnings("deprecation") + @Override + public Result search(Query query, Execution execution) { + Result result=execution.search(query); + for (Searcher branch : branches) { + Query branchQuery=parallel ? query.clone() : query; + Result branchResult= + ( carryOverContext ? new Execution(branch,execution.context()) : new Execution(branch, Execution.Context.createContextStub())).search(branchQuery); + result.hits().add(branchResult.hits()); + result.mergeWith(branchResult); + } + return result; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchchain/test/VespaAsyncSearcherTest.java b/container-search/src/test/java/com/yahoo/search/searchchain/test/VespaAsyncSearcherTest.java new file mode 100644 index 00000000000..954290de6a2 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchchain/test/VespaAsyncSearcherTest.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchchain.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.AsyncExecution; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.FutureResult; +import java.util.ArrayList; +import java.util.List; + +import junit.framework.TestCase; + +/** + * Externally provided test for async execution of search chains. + * + * @author <a href="mailto:pthomas@yahoo-inc.com">Peter Thomas</a> + */ +public class VespaAsyncSearcherTest extends TestCase { + private static class FirstSearcher extends Searcher { + + @Override + public Result search(Query query, Execution exctn) { + int count = 10; + List<FutureResult> futures = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + Query subQuery = new Query(); + FutureResult future = new AsyncExecution(exctn) + .search(subQuery); + futures.add(future); + } + AsyncExecution.waitForAll(futures, 10 * 60 * 1000); + return new Result(query); + } + + } + + private static class SecondSearcher extends Searcher { + + @Override + public Result search(Query query, Execution exctn) { + return new Result(query); + } + + } + + public void testAsyncExecution() { + Chain<Searcher> chain = new Chain<>(new FirstSearcher(), + new SecondSearcher()); + Execution execution = new Execution(chain, + Execution.Context.createContextStub(null)); + Query query = new Query(); + // fails with exception on old versions + execution.search(query); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java new file mode 100644 index 00000000000..b112e61cb44 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/CacheControlSearcherTestCase.java @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.CacheControlSearcher; +import junit.framework.TestCase; + +import java.util.List; + +import static com.yahoo.search.searchers.CacheControlSearcher.CACHE_CONTROL_HEADER; + +/** + * Unit test cases for CacheControlSearcher. + * + * @author <a href="http://techyard.corp.yahoo-inc.com/en/user/frodelu">Frode Lundgren</a> + */ +@SuppressWarnings("deprecation") +public class CacheControlSearcherTestCase extends TestCase { + + private Searcher getDocSource() { + return new Searcher() { + public Result search(Query query, Execution execution) { + Result res = new Result(query); + res.setTotalHitCount(1); + Hit hit = new Hit("http://document/", 1000); + hit.setField("url", "http://document/"); + hit.setField("title", "Article title"); + hit.setField("extsourceid", "12345"); + res.hits().add(hit); + return res; + } + }; + } + + private Chain<Searcher> getSearchChain() { + return new Chain<>(new CacheControlSearcher(), getDocSource()); + } + + private List<String> getCacheControlHeaders(Result result) { + return result.getHeaders(true).get(CACHE_CONTROL_HEADER); + } + + /** + * Assert that cache header ListMap exactly match given array of expected cache headers + * @param values - Array of cache control headers expected, e.g. {"max-age=120", "stale-while-revalidate=3600"} + * @param cacheheaders - The "Cache-Control" headers from the response ListMap + */ + private void assertCacheHeaders(String[] values, List<String> cacheheaders) { + assertNotNull("No headers to test for (was null)", values); + assertTrue("No headers to test for (no elements in array)", values.length > 0); + assertNotNull("No cache headers set in response", cacheheaders); + assertEquals(values.length, cacheheaders.size()); + for (String header : values) { + assertTrue("Cache header does not contain header '" + header + "'", cacheheaders.contains(header)); + } + } + + public void testNoHeader() { + Chain<Searcher> chain = getSearchChain(); + Query query = new Query("?query=foo&custid=foo"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertEquals(0, getCacheControlHeaders(result).size()); + } + + public void testInvalidAgeParams() { + Chain<Searcher> chain = getSearchChain(); + + try { + Query query = new Query("?query=foo&custid=foo&cachecontrol.maxage=foo"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertEquals(0, getCacheControlHeaders(result).size()); + fail("Expected exception"); + } + catch (NumberFormatException e) { + // success + } + + try { + Query query = new Query("?query=foo&custid=foo&cachecontrol.staleage=foo"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertEquals(0, getCacheControlHeaders(result).size()); + fail("Expected exception"); + } + catch (NumberFormatException e) { + // success + } + } + + public void testMaxAge() { + Chain<Searcher> chain = getSearchChain(); + + Query query = new Query("?query=foo&custid=foo&cachecontrol.maxage=120"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"max-age=120"}, getCacheControlHeaders(result)); + } + + public void testNoCache() { + Chain<Searcher> chain = getSearchChain(); + + Query query = new Query("?query=foo&custid=foo&cachecontrol.maxage=120&noCache"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"no-cache"}, getCacheControlHeaders(result)); + + query = new Query("?query=foo&custid=foo&cachecontrol.maxage=120&cachecontrol.nocache=true"); + result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"no-cache"}, getCacheControlHeaders(result)); + } + + public void testStateWhileRevalidate() { + Chain<Searcher> chain = getSearchChain(); + + Query query = new Query("?query=foo&custid=foo&cachecontrol.staleage=3600"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"stale-while-revalidate=3600"}, getCacheControlHeaders(result)); + } + + public void testStaleAndMaxAge() { + Chain<Searcher> chain = getSearchChain(); + + Query query = new Query("?query=foo&custid=foo&cachecontrol.maxage=60&cachecontrol.staleage=3600"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + assertCacheHeaders(new String[]{"max-age=60", "stale-while-revalidate=3600"}, getCacheControlHeaders(result)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/ConnectionControlSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/ConnectionControlSearcherTestCase.java new file mode 100644 index 00000000000..5e8596ef16d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/ConnectionControlSearcherTestCase.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import static org.junit.Assert.*; + +import java.io.ByteArrayInputStream; +import java.net.SocketAddress; +import java.net.URI; +import java.net.URISyntaxException; + +import org.junit.Test; +import org.mockito.Mockito; + +import com.yahoo.component.chain.Chain; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.Container; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.jdisc.http.HttpRequest.Version; +import com.yahoo.jdisc.service.CurrentContainer; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.ConnectionControlSearcher; + +/** + * Functionality tests for + * {@link com.yahoo.search.searchers.ConnectionControlSearcher}. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ConnectionControlSearcherTestCase { + + @Test + public final void test() throws URISyntaxException { + URI uri = new URI("http://finance.yahoo.com/?connectioncontrol.maxlifetime=1"); + long connectedAtMillis = 0L; + long nowMillis = 2L * 1000L; + Result r = doSearch(uri, connectedAtMillis, nowMillis); + assertEquals("Close", r.getHeaders(false).get("Connection").get(0)); + } + + @Test + public final void testForcedClose() throws URISyntaxException { + URI uri = new URI("http://finance.yahoo.com/?connectioncontrol.maxlifetime=0"); + long connectedAtMillis = 0L; + long nowMillis = 0L; + Result r = doSearch(uri, connectedAtMillis, nowMillis); + assertEquals("Close", r.getHeaders(false).get("Connection").get(0)); + } + + @Test + public final void testNormalCloseWithoutJdisc() { + long nowMillis = 2L; + Query query = new Query("/?connectioncontrol.maxlifetime=1"); + Execution e = new Execution(new Chain<Searcher>(ConnectionControlSearcher.createTestInstance(() -> nowMillis)), + Execution.Context.createContextStub()); + Result r = e.search(query); + assertNull(r.getHeaders(false)); + } + + @Test + public final void testNoMaxLifetime() throws URISyntaxException { + URI uri = new URI("http://finance.yahoo.com/"); + long connectedAtMillis = 0L; + long nowMillis = 0L; + Result r = doSearch(uri, connectedAtMillis, nowMillis); + assertNull(r.getHeaders(false)); + } + + @Test + public final void testYoungEnoughConnection() throws URISyntaxException { + URI uri = new URI("http://finance.yahoo.com/?connectioncontrol.maxlifetime=1"); + long connectedAtMillis = 0L; + long nowMillis = 500L; + Result r = doSearch(uri, connectedAtMillis, nowMillis); + assertNull(r.getHeaders(false)); + } + + + private Result doSearch(URI uri, long connectedAtMillis, long nowMillis) { + SocketAddress remoteAddress = Mockito.mock(SocketAddress.class); + Version version = Version.HTTP_1_1; + Method method = Method.GET; + CurrentContainer container = Mockito.mock(CurrentContainer.class); + Mockito.when(container.newReference(Mockito.any())).thenReturn(Mockito.mock(Container.class)); + final com.yahoo.jdisc.http.HttpRequest serverRequest = com.yahoo.jdisc.http.HttpRequest + .newServerRequest(container, uri, method, version, remoteAddress, connectedAtMillis); + HttpRequest incoming = new HttpRequest(serverRequest, new ByteArrayInputStream(new byte[0])); + Query query = new Query(incoming); + Execution e = new Execution(new Chain<Searcher>(ConnectionControlSearcher.createTestInstance(() -> nowMillis)), + Execution.Context.createContextStub()); + Result r = e.search(query); + return r; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java new file mode 100644 index 00000000000..ff521da1aad --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/InputCheckingSearcherTestCase.java @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import static org.junit.Assert.*; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.metrics.simple.MetricReceiver; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.InputCheckingSearcher; +import com.yahoo.text.Utf8; + +/** + * Functional test for InputCheckingSearcher. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class InputCheckingSearcherTestCase { + + Execution execution; + + @Before + public void setUp() throws Exception { + execution = new Execution(new Chain<Searcher>(new InputCheckingSearcher(MetricReceiver.nullImplementation)), + Execution.Context.createContextStub(new IndexFacts())); + } + + @After + public void tearDown() throws Exception { + execution = null; + } + + @Test + public final void testCommonCase() { + Result r = execution.search(new Query("/search/?query=three+blind+mice")); + assertNull(r.hits().getErrorHit()); + } + + @Test + public final void candidateButAsciiOnly() { + Result r = execution.search(new Query("/search/?query=a+a+a+a+a+a")); + assertNull(r.hits().getErrorHit()); + } + + @Test + public final void candidateButValid() throws UnsupportedEncodingException { + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("å å å å å å", "UTF-8"))); + assertNull(r.hits().getErrorHit()); + } + + @Test + public final void candidateButValidAndOutsideFirst256() throws UnsupportedEncodingException { + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode("œ œ œ œ œ œ", "UTF-8"))); + assertNull(r.hits().getErrorHit()); + } + + + @Test + public final void testDoubleEncoded() throws UnsupportedEncodingException { + String rawQuery = "å å å å å å"; + byte[] encodedOnce = Utf8.toBytes(rawQuery); + char[] secondEncodingBuffer = new char[encodedOnce.length]; + for (int i = 0; i < secondEncodingBuffer.length; ++i) { + secondEncodingBuffer[i] = (char) (encodedOnce[i] & 0xFF); + } + String query = new String(secondEncodingBuffer); + Result r = execution.search(new Query("/search/?query=" + URLEncoder.encode(query, "UTF-8"))); + assertEquals(1, r.hits().getErrorHit().errors().size()); + } + + @Test + public final void testRepeatedConsecutiveTermsInPhrase() { + Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + assertNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + assertNotNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); + assertNull(r.hits().getErrorHit()); + } + @Test + public final void testThatMaxRepeatedConsecutiveTermsInPhraseIs5() { + Result r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.c")); + assertNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=a.b.0.0.0.0.0.0.c")); + assertNotNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=a.b.0.0.0.1.0.0.0.c")); + assertNull(r.hits().getErrorHit()); + } + @Test + public final void testThatMaxRepeatedTermsInPhraseIs10() { + Result r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.9.a")); + assertNull(r.hits().getErrorHit()); + r = execution.search(new Query("/search/?query=0.a.1.a.2.a.3.a.4.a.5.a.6.a.7.a.8.a.9.a.10.a")); + assertNotNull(r.hits().getErrorHit()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java b/container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java new file mode 100644 index 00000000000..aaad8ba80ae --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/MockMetric.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.jdisc.Metric; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** +* @author bratseth +*/ +class MockMetric implements Metric { + + private Map<Context, Map<String, Number>> metrics = new HashMap<>(); + + public Map<String, Number> values(Context context) { + return metricsForContext(context); + } + + @Override + public void set(String key, Number val, Context context) { + metricsForContext(context).put(key, val); + } + + @Override + public void add(String key, Number value, Context context) { + Number previousValue = metricsForContext(context).get(key); + if (previousValue == null) + previousValue = 0; + metricsForContext(context).put(key, value.doubleValue() + previousValue.doubleValue()); + } + + /** Returns the metrics for a given context, never null */ + private Map<String, Number> metricsForContext(Context context) { + Map<String, Number> metricsForContext = metrics.get(context); + if (metricsForContext == null) { + metricsForContext = new HashMap<>(); + metrics.put(context, metricsForContext); + } + return metricsForContext; + } + + @Override + public Context createContext(Map<String, ?> dimensions) { + return new MapContext(dimensions); + } + + /** Creates a context containing a single dimension */ + public Metric.Context createContext(String dimensionName, String dimensionValue) { + if (dimensionName.isEmpty()) + return createContext(Collections.emptyMap()); + return createContext(Collections.singletonMap(dimensionName, dimensionValue)); + } + + private class MapContext implements Metric.Context { + + private final Map<String, ?> dimensions; + + public MapContext(Map<String, ?> dimensions) { + this.dimensions = dimensions; + } + + @Override + public int hashCode() { + return dimensions.hashCode(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if ( ! (o instanceof MapContext)) return false; + return dimensions.equals(((MapContext)o).dimensions); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingBenchmark.java b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingBenchmark.java new file mode 100644 index 00000000000..9381cf2ab7e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingBenchmark.java @@ -0,0 +1,208 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.cloud.config.ClusterInfoConfig; +import com.yahoo.component.chain.Chain; +import com.yahoo.jdisc.Metric; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.config.RateLimitingConfig; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.RateLimitingSearcher; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +/** + * A benchmark and multithread stress test of rate limiting. + * The purpose of this is to simulate the environment the rate limiter will work under in production + * and verify that it manages to keep rates more or less within set bounds and does not lead to excessive contention. + * + * @author bratseth + */ +public class RateLimitingBenchmark { + + private final int clientCount = 10; + private final int threadCount = 250; + private final int epochs = 100; // the number of times the sequence of load types are repeated + private final int totalQueriesPerThread = 4 * 1000 * 10; + + // This number produces a theoretical max request rate of 1000/5*threadCount = 50 k rps + // which in practice on my machine is about 40 k rps. + // With the number set to 0 my machine does about 150 k rps. + // This means that peaks (when it is zero) are roughly 3x base. + private final int sleepMsBetweenRequests = 5; + private final int peakDurationMs = 1000; + private final int timeBetweenPeaksMs = 2000; + + private final Chain<Searcher> chain; + private final MockMetric metric; + + private final Map<String, RequestCounts> requestCounters = new HashMap<>(); + + public RateLimitingBenchmark() { + RateLimitingConfig.Builder rateLimitingConfig = new RateLimitingConfig.Builder(); + /* Defaults: + rateLimitingConfig.maxAvailableCapacity(10000); + rateLimitingConfig.capacityIncrement(1000); + rateLimitingConfig.recheckForCapacityProbability(0.001); + */ + + rateLimitingConfig.maxAvailableCapacity(10000); + rateLimitingConfig.capacityIncrement(1000); + rateLimitingConfig.recheckForCapacityProbability(0.001); + + ClusterInfoConfig.Builder clusterInfoConfig = new ClusterInfoConfig.Builder(); + clusterInfoConfig.clusterId("testCluster"); + clusterInfoConfig.nodeCount(1); + + this.metric = new MockMetric(); + + chain = new Chain<>("test", new RateLimitingSearcher(new RateLimitingConfig(rateLimitingConfig), + new ClusterInfoConfig(clusterInfoConfig), metric)); + + for (int i = 0; i < clientCount ; i++) + requestCounters.put(toClientId(i), new RequestCounts()); + } + + public void run() throws InterruptedException { + long startTime = System.currentTimeMillis(); + runWorkers(); + long totalTime = Math.max(1, System.currentTimeMillis() - startTime); + + double totalAttemptedRate = 0; + for (int i=0; i < clientCount; i++) { + double attemptedRate = requestCounters.get(toClientId(i)).attempted.get() * 1000d / totalTime; + double allowedRate = requestCounters.get(toClientId(i)).allowed.get() * 1000d / totalTime; + System.out.println(String.format(Locale.ENGLISH, + "Client %1$2d: Attempted rate: %2$10.2f. Target allowed rate: %3$10.2f. Allowed rate: %4$10.2f. Rejected requests: %5$8d", + i, attemptedRate, Math.pow(4, i), allowedRate, rejectedRequests(i))); + totalAttemptedRate += attemptedRate; + } + System.out.println(String.format(Locale.ENGLISH, "\nTotal attempted rate: %1$10.2f seconds", totalAttemptedRate)); + System.out.println(String.format(Locale.ENGLISH, "\nTotal time: %1$8.2f seconds", totalTime/1000.0)); + } + + private void runWorkers() { + try { + long startTime = System.currentTimeMillis(); + + Thread[] threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; i++) + threads[i] = new Thread(new Worker(startTime)); + + for (int i = 0; i < threadCount; i++) + threads[i].start(); + + for (int i = 0; i < threadCount; i++) + threads[i].join(); + } + catch (Exception e) { // not production code + throw new RuntimeException(e); + } + } + + private int rejectedRequests(int id) { + Metric.Context context = metric.createContext("id", toClientId(id)); + Number rejectedRequestsMetric = metric.values(context).get("requestsOverQuota"); + if (rejectedRequestsMetric == null) return 0; + return rejectedRequestsMetric.intValue(); + } + + private class Worker implements Runnable { + + private final int sequences = 5; + private final long startTime; + + public Worker(long startTime) { + this.startTime = startTime; + } + + @Override + public void run() { + try { + for (int i = 0; i < epochs; i++) { + issueRequests(this::pickClientFairly); + issueRequests(this::pickClientSkewedToLowerNumbers); + issueRequests(this::pickClientSkewedToHigherNumbers); + issueRequests(this::pickClientFairly); + issueRequests(this::pickClientSkewedToHigherNumbers); + } + } + catch (InterruptedException e) { + // just end + } + } + + private void issueRequests(Supplier<Integer> clientNumberSupplier) throws InterruptedException { + for (int i = 0; i< totalQueriesPerThread/(epochs * sequences); i++) { + int clientNumber = clientNumberSupplier.get(); + requestCounters.get(toClientId(clientNumber)).addRequest(executeWasAllowed(chain, clientNumber)); + if ( ! isInPeak()) + Thread.sleep(sleepMsBetweenRequests); + } + } + + private boolean isInPeak() { + long timeSinceStart = System.currentTimeMillis() - startTime; + return timeSinceStart % timeBetweenPeaksMs < peakDurationMs; // a peak is at every start of every timeBetweenPeaks interval + } + + protected int pickClientFairly() { + return ThreadLocalRandom.current().nextInt(clientCount); + } + + protected int pickClientSkewedToLowerNumbers() { + int nr = (int)Math.floor((Math.pow(ThreadLocalRandom.current().nextDouble(), 3) * clientCount)); + if (nr > clientCount-1) return clientCount-1; + return nr; + } + + protected int pickClientSkewedToHigherNumbers() { + int nr = (int)Math.floor( ( 1- Math.pow(ThreadLocalRandom.current().nextDouble(), 3)) * clientCount); + if (nr > clientCount-1) return clientCount-1; + return nr; + } + + } + + private String toClientId(int n) { + return "id" + n; + } + + private boolean executeWasAllowed(Chain<Searcher> chain, int id) { + Query query = new Query(); + query.properties().set("rate.id", toClientId(id)); + query.properties().set("rate.cost", 1); + query.properties().set("rate.quota", Math.pow(4, id)); + query.properties().set("rate.idDimension", "id"); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + if (result.hits().getError() != null && result.hits().getError().getCode() == 429) + return false; + else + return true; + } + + + public static void main(String[] args) throws InterruptedException { + new RateLimitingBenchmark().run(); + } + + private static class RequestCounts { + + private AtomicInteger attempted = new AtomicInteger(0); + private AtomicInteger allowed = new AtomicInteger(0); + + public void addRequest(boolean wasAllowed) { + attempted.incrementAndGet(); + if (wasAllowed) allowed.incrementAndGet(); + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java new file mode 100755 index 00000000000..02d6620df2e --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/RateLimitingSearcherTestCase.java @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.cloud.config.ClusterInfoConfig; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.config.RateLimitingConfig; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.RateLimitingSearcher; +import com.yahoo.yolean.chain.After; +import org.junit.Test; +import com.yahoo.test.ManualClock; + +import java.time.Duration; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * Unit tests for RateLimitingSearcher + * + * @author bratseth + */ +public class RateLimitingSearcherTestCase { + + @Test + public void testRateLimiting() { + RateLimitingConfig.Builder rateLimitingConfig = new RateLimitingConfig.Builder(); + rateLimitingConfig.maxAvailableCapacity(4); + rateLimitingConfig.capacityIncrement(2); + rateLimitingConfig.recheckForCapacityProbability(1.0); + + ClusterInfoConfig.Builder clusterInfoConfig = new ClusterInfoConfig.Builder(); + clusterInfoConfig.clusterId("testCluster"); + clusterInfoConfig.nodeCount(4); + + ManualClock clock = new ManualClock(); + MockMetric metric = new MockMetric(); + + Chain<Searcher> chain = new Chain<Searcher>("test", new RateLimitingSearcher(new RateLimitingConfig(rateLimitingConfig), + new ClusterInfoConfig(clusterInfoConfig), + metric, clock), + new CostSettingSearcher()); + assertEquals("'rate' request are available initially", 2, tryRequests(chain, "id1")); + assertTrue("However, don't reject if we dryRun", executeWasAllowed(chain, "id1", true)); + clock.advance(Duration.ofMillis(1500)); // causes 2 new requests to become available + assertEquals("'rate' new requests became available", 2, tryRequests(chain, "id1")); + + assertEquals("Another id", 2, tryRequests(chain, "id2")); + + clock.advance(Duration.ofMillis(1000000)); + assertEquals("'maxAvailableCapacity' request became available", 4, tryRequests(chain, "id2")); + + assertFalse("If quota is set to 0, all requests are rejected, even initially", executeWasAllowed(chain, "id3", 0)); + + clock.advance(Duration.ofMillis(1000000)); + assertTrue("A single query which costs more than capacity is allowed as cost is calculated after allowing it", + executeWasAllowed(chain, "id1", 8, 8, false)); + assertFalse("capacity is -4: disallowing", executeWasAllowed(chain, "id1")); + clock.advance(Duration.ofMillis(1000)); + assertFalse("capacity is -2: disallowing", executeWasAllowed(chain, "id1")); + clock.advance(Duration.ofMillis(1000)); + assertFalse("capacity is 0: disallowing", executeWasAllowed(chain, "id1")); + clock.advance(Duration.ofMillis(1000)); + assertTrue(executeWasAllowed(chain, "id1")); + + // check metrics + assertEquals((double)requestsToTry-2 + 1 + requestsToTry-2 + 3, metric.values(metric.createContext("id", "id1")).get("requestsOverQuota")); + assertEquals((double)requestsToTry-2 + requestsToTry-4, metric.values(metric.createContext("id", "id2")).get("requestsOverQuota")); + } + + private int requestsToTry = 50; + + /** + * Try many requests and return how many was allowed. + * This is to avoid testing the exact pattern of request/deny which does not matter + * and is determined by floating point arithmetic details when capacity is close to zero. + */ + private int tryRequests(Chain<Searcher> chain, String id) { + int allowedCount = 0; + for (int i = 0; i < requestsToTry; i++) { + if (executeWasAllowed(chain, id)) + allowedCount++; + } + return allowedCount; + } + + private boolean executeWasAllowed(Chain<Searcher> chain, String id) { + return executeWasAllowed(chain, id, 8); // allowed 8 requests per second over 4 nodes -> 2 per node + } + + private boolean executeWasAllowed(Chain<Searcher> chain, String id, boolean dryRun) { + return executeWasAllowed(chain, id, 8, 1, dryRun); + } + + private boolean executeWasAllowed(Chain<Searcher> chain, String id, int quota) { + return executeWasAllowed(chain, id, quota, 1, false); + } + + private boolean executeWasAllowed(Chain<Searcher> chain, String id, double quota, double cost, boolean dryRun) { + Query query = new Query(); + query.properties().set("rate.id", id); + query.properties().set("cost", cost); // converted to rate.cost by a searcher executing after rate limiting + query.properties().set("rate.quota", quota); + query.properties().set("rate.idDimension", "id"); + query.properties().set("rate.dryRun", dryRun); + Result result = new Execution(chain, Execution.Context.createContextStub()).search(query); + if (result.hits().getError() != null && result.hits().getError().getCode() == 429) + return false; + else + return true; + } + + /** The purpose of this test is simply to verify that cost is picked up after executing the query */ + @After(RateLimitingSearcher.RATE_LIMITING) + private static class CostSettingSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + query.properties().set("rate.cost", query.properties().get("cost")); + return result; + } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java new file mode 100644 index 00000000000..4f7654ba3c0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.RawSource; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.search.Searcher; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchers.ValidateMatchPhaseSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.vespa.config.search.AttributesConfig; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * @author balder + */ +public class ValidateMatchPhaseSearcherTestCase { + + private ValidateMatchPhaseSearcher searcher; + + public ValidateMatchPhaseSearcherTestCase() { + searcher = new ValidateMatchPhaseSearcher( + ConfigGetter.getConfig(AttributesConfig.class, + "raw:", + new RawSource("attribute[4]\n" + + "attribute[0].name ok\n" + + "attribute[0].datatype INT32\n" + + "attribute[0].collectiontype SINGLE\n" + + "attribute[0].fastsearch true\n" + + "attribute[1].name not_fast\n" + + "attribute[1].datatype INT32\n" + + "attribute[1].collectiontype SINGLE\n" + + "attribute[1].fastsearch false\n" + + "attribute[2].name not_numeric\n" + + "attribute[2].datatype STRING\n" + + "attribute[2].collectiontype SINGLE\n" + + "attribute[2].fastsearch true\n" + + "attribute[3].name not_single\n" + + "attribute[3].datatype INT32\n" + + "attribute[3].collectiontype ARRAY\n" + + "attribute[3].fastsearch true" + ))); + } + + private static String getErrorMatch(String attribute) { + return "4: Invalid query parameter: The attribute '" + + attribute + + "' is not available for match-phase. It must be a single value numeric attribute with fast-search."; + } + + private static String getErrorDiversity(String attribute) { + return "4: Invalid query parameter: The attribute '" + + attribute + + "' is not available for match-phase diversification. It must be a single value numeric or string attribute."; + } + + @Test + public void testMatchPhaseAttribute() { + assertEquals("", search("")); + assertEquals("", match("ok")); + assertEquals(getErrorMatch("not_numeric"), match("not_numeric")); + assertEquals(getErrorMatch("not_single"), match("not_single")); + assertEquals(getErrorMatch("not_fast"), match("not_fast")); + assertEquals(getErrorMatch("not_found"), match("not_found")); + } + + @Test + public void testDiversityAttribute() { + assertEquals("", search("")); + assertEquals("", diversify("ok")); + assertEquals("", diversify("not_numeric")); + assertEquals(getErrorDiversity("not_single"), diversify("not_single")); + assertEquals("", diversify("not_fast")); + assertEquals(getErrorDiversity("not_found"), diversify("not_found")); + } + + private String match(String m) { + return search("&ranking.matchPhase.attribute=" + m); + } + + private String diversify(String m) { + return search("&ranking.matchPhase.attribute=ok&ranking.matchPhase.diversity.attribute=" + m); + } + + private String search(String m) { + String q = "/?query=sddocname:test" + m; + Result r = doSearch(searcher, new Query(q), 0, 10); + if (r.hits().getError() != null) { + return r.hits().getError().toString(); + } + return ""; + } + + private Result doSearch(Searcher searcher, Query query, int offset, int hits) { + query.setOffset(offset); + query.setHits(hits); + return createExecution(searcher).search(query); + } + + private Execution createExecution(Searcher searcher) { + Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(), new SimpleLinguistics()); + return new Execution(chainedAsSearchChain(searcher), context); + } + + private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { + List<Searcher> searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java new file mode 100644 index 00000000000..43563e29218 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/statistics/ElapsedTimeTestCase.java @@ -0,0 +1,433 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.statistics; + +import junit.framework.TestCase; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.statistics.ElapsedTime; +import com.yahoo.search.statistics.TimeTracker; +import com.yahoo.search.statistics.TimeTracker.Activity; +import com.yahoo.search.statistics.TimeTracker.SearcherTimer; + +/** + * Check sanity of TimeTracker and ElapsedTime. + * + * @author <a href="steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ElapsedTimeTestCase extends TestCase { + + private static final long[] SEARCH_TIMESEQUENCE = new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L }; + + private static final long[] SEARCH_AND_FILL_TIMESEQUENCE = new long[] { 1L, 2L, 3L, 4L, 5L, 6L, 7L, + // and here we start filling + 7L, 8L, 9L, 10L, 11L, 12L, 13L }; + + public static class CreativeTimeSource extends TimeTracker.TimeSource { + private int nowIndex = 0; + private long[] now; + + public CreativeTimeSource(long[] now) { + this.now = now; + } + + @Override + long now() { + long present = now[nowIndex++]; + if (present == 0L) { + // defensive coding against the innards of TimeTracker + throw new IllegalStateException("0 is an unsupported time stamp value."); + } + return present; + } + + } + + public static class UselessSearcher extends Searcher { + public UselessSearcher(String name) { + super(new ComponentId(name)); + } + + @Override + public Result search(Query query, Execution execution) { + return execution.search(query); + } + } + + private static class AlmostUselessSearcher extends Searcher { + AlmostUselessSearcher(String name) { + super(new ComponentId(name)); + } + + @Override + public Result search(Query query, Execution execution) { + Result r = execution.search(query); + Hit h = new Hit("nalle"); + h.setFillable(); + r.hits().add(h); + return r; + } + } + + private static class NoForwardSearcher extends Searcher { + @Override + public Result search(Query query, Execution execution) { + Result r = new Result(query); + Hit h = new Hit("nalle"); + h.setFillable(); + r.hits().add(h); + return r; + } + } + + private class TestingSearcher extends Searcher { + @Override + public Result search(Query query, Execution execution) { + Execution exec = new Execution(execution); + exec.timer().injectTimeSource( + new CreativeTimeSource(SEARCH_TIMESEQUENCE)); + exec.context().setDetailedDiagnostics(true); + Result r = exec.search(new Query()); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + assertNull(searchers[0].getInvoking(Activity.SEARCH)); + checkTiming(searchers, 1); + return r; + } + } + + private class SecondTestingSearcher extends Searcher { + @Override + public Result search(Query query, Execution execution) { + Execution exec = new Execution(execution); + exec.timer().injectTimeSource( + new CreativeTimeSource(SEARCH_AND_FILL_TIMESEQUENCE)); + exec.context().setDetailedDiagnostics(true); + Result result = exec.search(new Query()); + exec.fill(result); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + assertNull(searchers[0].getInvoking(Activity.SEARCH)); + checkTiming(searchers, 1); + assertNull(searchers[0].getInvoking(Activity.FILL)); + checkFillTiming(searchers, 1); + return result; + } + } + + private class ShortChainTestingSearcher extends Searcher { + @Override + public Result search(Query query, Execution execution) { + Execution exec = new Execution(execution); + exec.timer().injectTimeSource( + new CreativeTimeSource(new long[] { 1L, 2L, 2L })); + exec.context().setDetailedDiagnostics(true); + Result result = exec.search(new Query()); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + assertNull(searchers[0].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1].getInvoking(Activity.SEARCH)); + assertNull(searchers[1].getReturning(Activity.SEARCH)); + assertNull(searchers[0].getInvoking(Activity.FILL)); + assertNull(searchers[1].getInvoking(Activity.FILL)); + assertTrue(0 < result.getElapsedTime().detailedReport().indexOf("NoForwardSearcher")); + return result; + } + } + + public void testBasic() { + TimeTracker t = new TimeTracker(null); + t.injectTimeSource(new CreativeTimeSource(new long[] {1L, 2L, 3L, 4L})); + Query q = new Query(); + Result r = new Result(q); + t.sampleSearch(0, false); + t.sampleFill(0, false); + t.samplePing(0, false); + t.sampleSearchReturn(0, false, r); + assertEquals(1L, t.first()); + assertEquals(4L, t.last()); + assertEquals(2L, t.firstFill()); + assertEquals(1L, t.searchTime()); + assertEquals(1L, t.fillTime()); + assertEquals(1L, t.pingTime()); + assertEquals(3L, t.totalTime()); + } + + public void testMultiSearchAndPing() { + TimeTracker t = new TimeTracker(null); + t.injectTimeSource(new CreativeTimeSource(new long[] {1L, 4L, 16L, 32L, 64L, 128L, 256L})); + Query q = new Query(); + Result r = new Result(q); + t.sampleSearch(0, false); + t.samplePing(0, false); + t.sampleSearch(0, false); + t.samplePing(0, false); + t.sampleSearch(0, false); + t.sampleFill(0, false); + t.sampleSearchReturn(0, false, r); + assertEquals(1L, t.first()); + assertEquals(256L, t.last()); + assertEquals(128L, t.firstFill()); + assertEquals(83L, t.searchTime()); + assertEquals(128L, t.fillTime()); + assertEquals(44L, t.pingTime()); + assertEquals(255L, t.totalTime()); + ElapsedTime e = new ElapsedTime(); + e.add(t); + e.add(t); + // multiple adds is supposed to be safe + assertEquals(255L, t.totalTime()); + TimeTracker tx = new TimeTracker(null); + tx.injectTimeSource(new CreativeTimeSource(new long[] {1L, 2L, 3L, 4L})); + Query qx = new Query(); + Result rx = new Result(qx); + tx.sampleSearch(0, false); + tx.sampleFill(0, false); + tx.samplePing(0, false); + tx.sampleSearchReturn(0, false, rx); + e.add(tx); + assertEquals(258L, e.totalTime()); + assertEquals(129L, e.fillTime()); + assertEquals(2L, e.firstFill()); + } + + public void testBasicBreakdown() { + TimeTracker t = new TimeTracker(new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + t.injectTimeSource(new CreativeTimeSource(new long[] { 1L, 2L, 3L, + 4L, 5L, 6L, 7L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + SearcherTimer[] searchers = t.searcherTracking(); + checkTiming(searchers); + } + + // This test is to make sure the other tests correctly simulate the call + // order into the TimeTracker + public void testBasicBreakdownFullyWiredIn() { + Chain<? extends Searcher> chain = new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third")); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.timer().injectTimeSource(new CreativeTimeSource(SEARCH_TIMESEQUENCE)); + exec.context().setDetailedDiagnostics(true); + exec.search(new Query()); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + checkTiming(searchers); + } + + + private void checkTiming(SearcherTimer[] searchers) { + checkTiming(searchers, 0); + } + + private void checkTiming(SearcherTimer[] searchers, int offset) { + assertEquals(Long.valueOf(1L), searchers[0 + offset].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1 + offset].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2 + offset].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2 + offset].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1 + offset].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[0 + offset].getReturning(Activity.SEARCH)); + } + + public void testBasicBreakdownWithFillFullyWiredIn() { + Chain<? extends Searcher> chain = new Chain<>( + new UselessSearcher("first"), new UselessSearcher("second"), + new AlmostUselessSearcher("third")); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.timer().injectTimeSource( + new CreativeTimeSource(SEARCH_AND_FILL_TIMESEQUENCE)); + exec.context().setDetailedDiagnostics(true); + Result result = exec.search(new Query()); + exec.fill(result); + SearcherTimer[] searchers = exec.timer().searcherTracking(); + checkTiming(searchers); + checkFillTiming(searchers); + } + + private void checkFillTiming(SearcherTimer[] searchers) { + checkFillTiming(searchers, 0); + } + + private void checkFillTiming(SearcherTimer[] searchers, int offset) { + assertEquals(Long.valueOf(1L), searchers[0 + offset].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[1 + offset].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[2 + offset].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[2 + offset].getReturning(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[1 + offset].getReturning(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[0 + offset].getReturning(Activity.FILL)); + } + + public void testBasicBreakdownFullyWiredInFirstSearcherNotFirstInChain() { + Chain<? extends Searcher> chain = new Chain<>( + new TestingSearcher(), + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third")); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.search(new Query()); + } + + public void testBasicBreakdownWithFillFullyWiredInFirstSearcherNotFirstInChain() { + Chain<? extends Searcher> chain = new Chain<>( + new SecondTestingSearcher(), + new UselessSearcher("first"), new UselessSearcher("second"), + new AlmostUselessSearcher("third")); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.search(new Query()); + } + + public void testTimingWithShortChain() { + Chain<? extends Searcher> chain = new Chain<>( + new ShortChainTestingSearcher(), + new NoForwardSearcher()); + Execution exec = new Execution(chain, Execution.Context.createContextStub()); + exec.search(new Query()); + } + + public void testBasicBreakdownReturnInsideSearchChain() { + TimeTracker t = new TimeTracker(new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + t.injectTimeSource(new CreativeTimeSource(new long[] { 1L, 2L, 3L, + 4L, 5L, 6L })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + SearcherTimer[] searchers = t.searcherTracking(); + assertEquals(Long.valueOf(1L), searchers[0].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2].getInvoking(Activity.SEARCH)); + assertNull(searchers[2].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L) ,searchers[1].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L) ,searchers[0].getReturning(Activity.SEARCH)); + } + + public void testBasicBreakdownWithFill() { + TimeTracker t = new TimeTracker(new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + t.injectTimeSource(new CreativeTimeSource(new long[] { 1L, 2L, 3L, + 4L, 5L, 6L, 7L, 7L, 8L, 9L, 10L})); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearch(3, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + t.sampleSearchReturn(0, true, null); + t.sampleFill(0, true); + t.sampleFill(1, true); + t.sampleFillReturn(1, true, null); + t.sampleFillReturn(0, true, null); + SearcherTimer[] searchers = t.searcherTracking(); + assertEquals(Long.valueOf(1L), searchers[0].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[2].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[1].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[0].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(1L), searchers[0].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[1].getInvoking(Activity.FILL)); + assertNull(searchers[1].getReturning(Activity.FILL)); + assertEquals(Long.valueOf(1L), searchers[0].getReturning(Activity.FILL)); + } + + + private void runSomeTraffic(TimeTracker t) { + t.injectTimeSource(new CreativeTimeSource(new long[] { + 1L, 2L, 3L, + // checkpoint 1 + 4L, 5L, + // checkpoint 2 + 6L, 7L, 8L, 9L, + // checkpoint 3 + 10L, 11L, 12L, 13L, + // checkpoint 4 + 14L, 15L, 16L, 17L, + // checkpoint 5 + 18L + })); + t.sampleSearch(0, true); + t.sampleSearch(1, true); + t.sampleSearch(2, true); + // checkpoint 1 + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + // checkpoint 2 + t.sampleFill(1, true); + t.sampleFill(2, true); + t.sampleFillReturn(2, true, null); + t.sampleFillReturn(1, true, null); + // checkpoint 3 + t.sampleSearch(1, true); + t.sampleSearch(2, true); + t.sampleSearchReturn(2, true, null); + t.sampleSearchReturn(1, true, null); + // checkpoint 4 + t.sampleFill(1, true); + t.sampleFill(2, true); + t.sampleFillReturn(2, true, null); + t.sampleFillReturn(1, true, null); + // checkpoint 5 + t.sampleSearchReturn(0, true, null); + } + + public void testMixedActivity() { + TimeTracker t = new TimeTracker(new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + runSomeTraffic(t); + + SearcherTimer[] searchers = t.searcherTracking(); + assertEquals(Long.valueOf(1L), searchers[0].getInvoking(Activity.SEARCH)); + assertNull(searchers[0].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(2L), searchers[0].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(2L), searchers[0].getReturning(Activity.FILL)); + + assertEquals(Long.valueOf(2L), searchers[1].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(2L), searchers[1].getInvoking(Activity.FILL)); + assertEquals(Long.valueOf(2L), searchers[1].getReturning(Activity.SEARCH)); + assertEquals(Long.valueOf(2L), searchers[1].getReturning(Activity.FILL)); + + assertEquals(Long.valueOf(2L), searchers[2].getInvoking(Activity.SEARCH)); + assertEquals(Long.valueOf(2L), searchers[2].getInvoking(Activity.FILL)); + assertNull(searchers[2].getReturning(Activity.SEARCH)); + assertNull(searchers[2].getReturning(Activity.FILL)); + } + + public void testReportGeneration() { + TimeTracker t = new TimeTracker(new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + runSomeTraffic(t); + + ElapsedTime elapsed = new ElapsedTime(); + elapsed.add(t); + t = new TimeTracker(new Chain<Searcher>( + new UselessSearcher("first"), new UselessSearcher("second"), + new UselessSearcher("third"))); + runSomeTraffic(t); + elapsed.add(t); + assertEquals(true, elapsed.hasDetailedData()); + assertEquals("Time use per searcher:" + + " first(QueryProcessing(SEARCH: 2 ms), ResultProcessing(SEARCH: 4 ms, FILL: 4 ms)),\n" + + " second(QueryProcessing(SEARCH: 4 ms, FILL: 4 ms), ResultProcessing(SEARCH: 4 ms, FILL: 4 ms)),\n" + + " third(QueryProcessing(SEARCH: 4 ms, FILL: 4 ms), ResultProcessing()).", + elapsed.detailedReport()); + } + + public static void doInjectTimeSource(TimeTracker t, TimeTracker.TimeSource s) { + t.injectTimeSource(s); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java new file mode 100644 index 00000000000..fba46e1dfbe --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/statistics/PeakQpsTestCase.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.statistics; + +import static org.junit.Assert.*; + +import java.util.Deque; +import java.util.List; + +import com.yahoo.statistics.Statistics; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.concurrent.LocalInstance; +import com.yahoo.concurrent.ThreadLocalDirectory; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.statistics.PeakQpsSearcher.QueryRatePerSecond; + +/** + * Check peak QPS aggregation has a chance of working. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class PeakQpsTestCase { + + static class Producer implements Runnable { + private final ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> rates; + + Producer(ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> rates) { + this.rates = rates; + } + + @Override + public void run() { + LocalInstance<Deque<QueryRatePerSecond>, Long> rate = rates.getLocalInstance(); + rates.update(1L, rate); + rates.update(2L, rate); + rates.update(2L, rate); + rates.update(3L, rate); + rates.update(3L, rate); + rates.update(3L, rate); + rates.update(4L, rate); + rates.update(4L, rate); + rates.update(4L, rate); + rates.update(4L, rate); + } + } + + static class LaterProducer implements Runnable { + private final ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> rates; + + LaterProducer(ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> rates) { + this.rates = rates; + } + + @Override + public void run() { + LocalInstance<Deque<QueryRatePerSecond>, Long> rate = rates.getLocalInstance(); + rates.update(2L, rate); + rates.update(2L, rate); + rates.update(3L, rate); + rates.update(3L, rate); + rates.update(3L, rate); + rates.update(5L, rate); + rates.update(5L, rate); + rates.update(6L, rate); + rates.update(7L, rate); + } + } + + @Test + public void checkBasicDataAggregation() { + ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> directory = PeakQpsSearcher.createDirectory(); + final int threadCount = 20; + Thread[] threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; ++i) { + Producer p = new Producer(directory); + threads[i] = new Thread(p); + threads[i].start(); + } + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + // nop + } + } + List<Deque<QueryRatePerSecond>> measurements = directory.fetch(); + List<QueryRatePerSecond> results = PeakQpsSearcher.merge(measurements); + assertTrue(results.get(0).when == 1L); + assertTrue(results.get(0).howMany == threadCount); + assertTrue(results.get(1).when == 2L); + assertTrue(results.get(1).howMany == threadCount * 2); + assertTrue(results.get(2).when == 3L); + assertTrue(results.get(2).howMany == threadCount * 3); + assertTrue(results.get(3).when == 4L); + assertTrue(results.get(3).howMany == threadCount * 4); + } + + @Test + public void checkMixedDataAggregation() { + ThreadLocalDirectory<Deque<QueryRatePerSecond>, Long> directory = PeakQpsSearcher.createDirectory(); + final int firstThreads = 20; + final int secondThreads = 20; + final int threadCount = firstThreads + secondThreads; + Thread[] threads = new Thread[threadCount]; + for (int i = 0; i < threadCount; ++i) { + if (i < firstThreads) { + Producer p = new Producer(directory); + threads[i] = new Thread(p); + } else { + LaterProducer p = new LaterProducer(directory); + threads[i] = new Thread(p); + } + threads[i].start(); + + } + for (Thread t : threads) { + try { + t.join(); + } catch (InterruptedException e) { + // nop + } + } + List<Deque<QueryRatePerSecond>> measurements = directory.fetch(); + List<QueryRatePerSecond> results = PeakQpsSearcher.merge(measurements); + assertTrue(results.size() == 7); + assertTrue(results.get(0).when == 1L); + assertTrue(results.get(0).howMany == firstThreads); + assertTrue(results.get(1).when == 2L); + assertTrue(results.get(1).howMany == threadCount * 2); + assertTrue(results.get(2).when == 3L); + assertTrue(results.get(2).howMany == threadCount * 3); + assertTrue(results.get(3).when == 4L); + assertTrue(results.get(3).howMany == firstThreads * 4); + assertTrue(results.get(4).when == 5L); + assertTrue(results.get(4).howMany == secondThreads * 2); + assertTrue(results.get(5).when == 6L); + assertTrue(results.get(5).howMany == secondThreads); + assertTrue(results.get(6).when == 7L); + assertTrue(results.get(6).howMany == secondThreads); + } + + @Test + public void checkSearch() { + MeasureQpsConfig config = new MeasureQpsConfig( + new MeasureQpsConfig.Builder().outputmethod( + MeasureQpsConfig.Outputmethod.METAHIT).queryproperty( + "qpsprobe")); + Searcher s = new PeakQpsSearcher(config, Statistics.nullImplementation); + Chain<Searcher> c = new Chain<>(s); + Execution e = new Execution(c, Execution.Context.createContextStub()); + e.search(new Query("/?query=a")); + new Execution(c, Execution.Context.createContextStub()); + Result r = e.search(new Query("/?query=a&qpsprobe=true")); + final Hit hit = r.hits().get(0); + assertTrue(hit instanceof PeakQpsSearcher.QpsHit); + assertNotNull(hit.fields().get(PeakQpsSearcher.QpsHit.MEAN_QPS)); + assertNotNull(hit.fields().get(PeakQpsSearcher.QpsHit.PEAK_QPS)); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java new file mode 100644 index 00000000000..2f086dbe5a8 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/statistics/TimingSearcherTestCase.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.statistics; + +import junit.framework.TestCase; + +import com.yahoo.component.ComponentId; +import com.yahoo.prelude.Ping; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.statistics.TimingSearcher.Parameters; +import com.yahoo.statistics.Statistics; +import com.yahoo.statistics.Value; + +public class TimingSearcherTestCase extends TestCase { + public static class MockValue extends Value { + public int putCount = 0; + + public MockValue() { + super("mock", Statistics.nullImplementation, new Value.Parameters()); + } + + @Override + public void put(double x) { + putCount += 1; + } + } + + public void testMeasurementSearchPath() { + Parameters p = new Parameters("timingtest", TimeTracker.Activity.SEARCH); + TimingSearcher ts = new TimingSearcher(new ComponentId("lblblbl"), p, Statistics.nullImplementation); + MockValue v = new MockValue(); + ts.setMeasurements(v); + Execution exec = new Execution(ts, Execution.Context.createContextStub()); + Result r = exec.search(new Query("/?query=a")); + Hit f = new Hit("blblbl"); + f.setFillable(); + r.hits().add(f); + exec.fill(r, "whatever"); + exec.fill(r, "lalala"); + exec.ping(new Ping()); + exec.ping(new Ping()); + exec.ping(new Ping()); + assertEquals(1, v.putCount); + } + + public void testMeasurementFillPath() { + Parameters p = new Parameters("timingtest", TimeTracker.Activity.FILL); + TimingSearcher ts = new TimingSearcher(new ComponentId("lblblbl"), p, Statistics.nullImplementation); + MockValue v = new MockValue(); + ts.setMeasurements(v); + Execution exec = new Execution(ts, Execution.Context.createContextStub()); + Result r = exec.search(new Query("/?query=a")); + Hit f = new Hit("blblbl"); + f.setFillable(); + r.hits().add(f); + exec.fill(r, "whatever"); + exec.fill(r, "lalala"); + exec.ping(new Ping()); + exec.ping(new Ping()); + exec.ping(new Ping()); + assertEquals(2, v.putCount); + } + + public void testMeasurementPingPath() { + Parameters p = new Parameters("timingtest", TimeTracker.Activity.PING); + TimingSearcher ts = new TimingSearcher(new ComponentId("lblblbl"), p, Statistics.nullImplementation); + MockValue v = new MockValue(); + ts.setMeasurements(v); + Execution exec = new Execution(ts, Execution.Context.createContextStub()); + Result r = exec.search(new Query("/?query=a")); + Hit f = new Hit("blblbl"); + f.setFillable(); + r.hits().add(f); + exec.fill(r, "whatever"); + exec.fill(r, "lalala"); + exec.ping(new Ping()); + exec.ping(new Ping()); + exec.ping(new Ping()); + assertEquals(3, v.putCount); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/statistics/test/.gitignore b/container-search/src/test/java/com/yahoo/search/statistics/test/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/statistics/test/.gitignore diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryBenchmark.java b/container-search/src/test/java/com/yahoo/search/test/QueryBenchmark.java new file mode 100644 index 00000000000..bab7c0be548 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/test/QueryBenchmark.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.test; + +import com.yahoo.search.Query; + +/** + * Tests the speed of accessing the query + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class QueryBenchmark { + + public void run() { + int result=0; + + // Warm-up + out("Warming up..."); + for (int i=0; i<10*1000; i++) + result+=createAndAccessQuery(i); + + long startTime=System.currentTimeMillis(); + out("Running..."); + for (int i=0; i<100*1000; i++) + result+=createAndAccessQuery(i); + out("Ignore this: " + result); // Make sure we are not fooled by optimization by creating an observable result + long endTime=System.currentTimeMillis(); + out("Creating and accessing a query 100.000 times took " + (endTime-startTime) + " ms"); + } + + private final int createAndAccessQuery(int i) { + // 8 sets, 8 gets + + Query query=new Query("?query=test&hits=10&presentation.bolding=true&model.type=all"); + query.properties().set("model.defaultIndex","title"); + query.properties().set("string1","value1:" + i); + query.properties().set("string2","value2:" + i); + query.properties().set("string3","value3:" + i); + int result=((String)query.properties().get("string1")).length(); + result+=((String)query.properties().get("string2")).length(); + result+=((String)query.properties().get("string3")).length(); + result+=((String)query.properties().get("model.defaultIndex")).length(); + + Query clone=query.clone(); + result+=((String)query.properties().get("string1")).length(); + result+=((String)query.properties().get("string2")).length(); + result+=((String)query.properties().get("string3")).length(); + result+=((String)clone.properties().get("model.defaultIndex")).length(); + return result; + } + + private void out(String string) { + System.out.println(string); + } + + public static void main(String[] args) { + new QueryBenchmark().run(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java new file mode 100644 index 00000000000..a9690fd1983 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -0,0 +1,671 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.Highlight; +import com.yahoo.prelude.query.IndexedItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +import com.yahoo.yolean.Exceptions; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.fail; + +/** + * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> + */ +public class QueryTestCase { + + @Test + public void testSimpleFunctionality() { + Query q = new Query(QueryTestCase.httpEncode("/sdfsd.html?query=this is a simple query&aParameter")); + assertEquals("this is a simple query", q.getModel().getQueryString()); + assertNotNull(q.getModel().getQueryTree()); + assertNull(q.getModel().getDefaultIndex()); + assertEquals("", q.properties().get("aParameter")); + assertNull(q.properties().get("notSetParameter")); + } + + // TODO: YQL work in progress (jon) + @Ignore + @Test + public void testSimpleProgram() { + Query q = new Query(httpEncode("?program=select * where myfield contains(word)")); + assertEquals("", q.getModel().getQueryTree().toString()); + } + + // TODO: YQL work in progress (jon) + @Ignore + @Test + public void testSimpleProgramParameterAlias() throws UnsupportedEncodingException { + Query q = new Query(httpEncode("/sdfsd.html?yql=select * from source where myfield contains(word);")); + assertEquals("", q.getModel().getQueryTree().toString()); + } + + @Test + public void testClone() { + Query q = new Query(httpEncode("/sdfsd.html?query=this+is+a+simple+query&aParameter")); + q.getPresentation().setHighlight(new Highlight()); + Query p = q.clone(); + assertEquals(q, p); + assertEquals(q.hashCode(), p.hashCode()); + + // Make sure we deep clone all mutable objects + + assertNotSame(q, p); + assertNotSame(q.getRanking(), p.getRanking()); + assertNotSame(q.getRanking().getFeatures(), p.getRanking().getFeatures()); + assertNotSame(q.getRanking().getProperties(), p.getRanking().getProperties()); + assertNotSame(q.getRanking().getMatchPhase(), p.getRanking().getMatchPhase()); + assertNotSame(q.getRanking().getMatchPhase().getDiversity(), p.getRanking().getMatchPhase().getDiversity()); + + assertNotSame(q.getPresentation(), p.getPresentation()); + assertNotSame(q.getPresentation().getHighlight(), p.getPresentation().getHighlight()); + assertNotSame(q.getPresentation().getSummaryFields(), p.getPresentation().getSummaryFields()); + + assertNotSame(q.getModel(), p.getModel()); + assertNotSame(q.getModel().getSources(), p.getModel().getSources()); + assertNotSame(q.getModel().getRestrict(), p.getModel().getRestrict()); + assertNotSame(q.getModel().getQueryTree(), p.getModel().getQueryTree()); + } + + private boolean isA(String s) { + return (s.equals("a")); + } + + private void printIt(List<String> l) { + System.out.println(l); + } + + @Test + public void testCloneWithConnectivity() { + List<String> l = new ArrayList(); + l.add("a"); + l.add("b"); + l.add("c"); + l.add("a"); + printIt(l.stream().filter(i -> isA(i)).collect(Collectors.toList())); + printIt(l.stream().filter(i -> ! isA(i)).collect(Collectors.toList())); + + Query q = new Query(); + WordItem a = new WordItem("a"); + WordItem b = new WordItem("b"); + WordItem c = new WordItem("c"); + WordItem d = new WordItem("d"); + WordItem e = new WordItem("e"); + WordItem f = new WordItem("f"); + WordItem g = new WordItem("g"); + + OrItem or = new OrItem(); + or.addItem(c); + or.addItem(d); + + AndItem and1 = new AndItem(); + and1.addItem(a); + and1.addItem(b); + and1.addItem(or); + and1.addItem(e); + + AndItem and2 = new AndItem(); + and2.addItem(f); + and2.addItem(g); + + RankItem rank = new RankItem(); + rank.addItem(and1); + rank.addItem(and2); + + a.setConnectivity(b, 0.1); + b.setConnectivity(c, 0.2); + c.setConnectivity(d, 0.3); + d.setConnectivity(e, 0.4); + e.setConnectivity(f, 0.5); + f.setConnectivity(g, 0.6); + + q.getModel().getQueryTree().setRoot(rank); + Query qClone = q.clone(); + assertEquals(q, qClone); + + RankItem rankClone = (RankItem)qClone.getModel().getQueryTree().getRoot(); + AndItem and1Clone = (AndItem)rankClone.getItem(0); + AndItem and2Clone = (AndItem)rankClone.getItem(1); + OrItem orClone = (OrItem)and1Clone.getItem(2); + + WordItem aClone = (WordItem)and1Clone.getItem(0); + WordItem bClone = (WordItem)and1Clone.getItem(1); + WordItem cClone = (WordItem)orClone.getItem(0); + WordItem dClone = (WordItem)orClone.getItem(1); + WordItem eClone = (WordItem)and1Clone.getItem(3); + WordItem fClone = (WordItem)and2Clone.getItem(0); + WordItem gClone = (WordItem)and2Clone.getItem(1); + + assertTrue(rankClone != rank); + assertTrue(and1Clone != and1); + assertTrue(and2Clone != and2); + assertTrue(orClone != or); + + assertTrue(aClone != a); + assertTrue(bClone != b); + assertTrue(cClone != c); + assertTrue(dClone != d); + assertTrue(eClone != e); + assertTrue(fClone != f); + assertTrue(gClone != g); + + assertTrue(aClone.getConnectedItem() == bClone); + assertTrue(bClone.getConnectedItem() == cClone); + assertTrue(cClone.getConnectedItem() == dClone); + assertTrue(dClone.getConnectedItem() == eClone); + assertTrue(eClone.getConnectedItem() == fClone); + assertTrue(fClone.getConnectedItem() == gClone); + + double delta = 0.0000001; + assertEquals(0.1, aClone.getConnectivity(), delta); + assertEquals(0.2, bClone.getConnectivity(), delta); + assertEquals(0.3, cClone.getConnectivity(), delta); + assertEquals(0.4, dClone.getConnectivity(), delta); + assertEquals(0.5, eClone.getConnectivity(), delta); + assertEquals(0.6, fClone.getConnectivity(), delta); + } + + @Test + public void test_that_cloning_preserves_timeout() { + Query original = new Query(); + original.setTimeout(9876l); + + Query clone = original.clone(); + assertThat(clone.getTimeout(), is(9876l)); + } + + @Test + public void testTimeout() { + // yes, this test depends on numbers which have exact IEEE representations + Query q = new Query(httpEncode("/search?timeout=500")); + assertEquals(500000L, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=500 ms")); + assertEquals(500, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=500.0ms")); + assertEquals(500, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=500.0s")); + assertEquals(500000, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=5ks")); + assertEquals(5000000, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q = new Query(httpEncode("/search?timeout=5000.0 \u00B5s")); + assertEquals(5, q.getTimeout()); + assertEquals(0, q.errors().size()); + + // seconds is unit when unknown unit + q = new Query(httpEncode("/search?timeout=42 yrs")); + assertEquals(42000, q.getTimeout()); + assertEquals(0, q.errors().size()); + + q=new Query(); + q.setTimeout(53L); + assertEquals(53L, q.properties().get("timeout")); + assertEquals(53L, q.getTimeout()); + + // This is the unfortunate consequence of this legacy: + q=new Query(); + q.properties().set("timeout", 53L); + assertEquals(53L * 1000, q.properties().get("timeout")); + assertEquals(53L * 1000, q.getTimeout()); + } + + @Test + public void testUnparseableTimeout() { + try { + new Query(httpEncode("/search?timeout=nalle")); + fail("Above statement should throw"); + } catch (QueryException e) { + // As expected. + assertThat( + Exceptions.toMessageString(e), + containsString("Could not set 'timeout' to 'nalle': Error parsing 'nalle': Invalid number 'nalle'")); + } + } + + @Test + public void testTimeoutInRequestOverridesQueryProfile() { + QueryProfile profile = new QueryProfile("test"); + profile.set("timeout", 318, (QueryProfileRegistry)null); + Query q = new Query(QueryTestCase.httpEncode("/search?timeout=500"), profile.compile(null)); + assertEquals(500000L, q.getTimeout()); + } + + @Test + public void testNotEqual() { + Query q = new Query("/?query=something+test&nocache"); + Query p = new Query("/?query=something+test"); + assertEquals(q,p); + assertEquals(q.hashCode(),p.hashCode()); + Query r = new Query("?query=something+test&hits=5"); + assertNotSame(q,r); + assertNotSame(q.hashCode(),r.hashCode()); + } + + @Test + public void testEqual() { + assertEquals(new Query("?query=12").hashCode(),new Query("?query=12").hashCode()); + assertEquals(new Query("?query=12"),new Query("?query=12")); + } + + @Test + public void testUtf8Decoding() { + Query q = new Query("/?query=beyonc%C3%A9"); + q.getModel().getQueryTree().toString(); + assertEquals("beyonc\u00e9", q.getModel().getQueryTree().toString()); + } + + @Test + public void testDefaultIndex() { + Query q = new Query("?query=hi%20hello%20keyword:kanoo%20" + + "default:munkz%20%22phrases+too%22&default-index=def"); + assertEquals("AND def:hi def:hello keyword:kanoo default:munkz def:\"phrases too\"", + q.getModel().getQueryTree().toString()); + } + + @Test + public void testHashCode() { + Query p = new Query("?query=foo&type=any"); + Query q = new Query("?query=foo&type=all"); + assertTrue(p.hashCode() != q.hashCode()); + } + + @Test + public void testSimpleQueryParsing () { + Query q = new Query("/search?query=foobar&offset=10&hits=20"); + assertEquals("foobar",q.getModel().getQueryTree().toString()); + assertEquals(10,q.getOffset()); + assertEquals(20,q.getHits()); + } + + /** Test that GET parameter names are case in-sensitive */ + @Test + public void testGETParametersCase() { + Query q = new Query("?QUERY=testing&hits=10&oFfSeT=10"); + assertEquals("testing", q.getModel().getQueryString()); + assertEquals(10, q.getHits()); + assertEquals(10, q.getOffset()); + } + + /** Test that we get the last value if a parameter is assigned multiple times */ + @Test + public void testRepeatedParameter() { + Query q = new Query("?query=test&hits=5&hits=10"); + assertEquals(10, q.getHits()); + } + + @Test + public void testNoCache() { + Query q = new Query("search?query=foobar&nocache"); + assertTrue(q.getNoCache()); + } + + @Test + public void testSessionCache() { + Query q = new Query("search?query=foobar&groupingSessionCache"); + assertTrue(q.getGroupingSessionCache()); + q = new Query("search?query=foobar"); + assertFalse(q.getGroupingSessionCache()); + } + + public class TestClass { + private int testInt = 0; + public int getTestInt() { + return testInt; + } + + public void setTestInt(int testInt) { + this.testInt = testInt; + } + + public void setTestInt(String testInt) { + this.testInt = Integer.parseInt(testInt); + } + } + + @Test + public void testSetting() { + Query q = new Query(); + q.properties().set("test", "test"); + assertEquals(q.properties().get("test"), "test"); + + TestClass tc = new TestClass(); + q.properties().set("test", tc); + assertEquals(q.properties().get("test"), tc); + q.properties().set("test.testInt", 1); + assertEquals(q.properties().get("test.testInt"), 1); + } + + @Test + public void testAlias() { + Query q = new Query("search?query=testing&language=en"); + assertEquals(q.getModel().getLanguage(), q.properties().get("model.language")); + } + + @Test + public void testTracing() { + Query q = new Query("?query=foo&traceLevel=2"); + assertEquals(2, q.getTraceLevel()); + q.trace(true, 1, "trace1"); + q.trace(false,2, "trace2"); + q.trace(true, 3, "Ignored"); + q.trace(true, 2, "trace3-1", ", ", "trace3-2"); + q.trace(false,1, "trace4-1", ", ", "trace4-2"); + q.trace(false,3, "Ignored-1", "Ignored-2"); + Set<String> traces = new HashSet<>(); + for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) + traces.add(trace); + // for (String s : traces) System.out.println(s); + assertTrue(traces.contains("trace1: [select * from sources * where default contains \"foo\";]")); + assertTrue(traces.contains("trace2")); + assertTrue(traces.contains("trace3-1, trace3-2: [select * from sources * where default contains \"foo\";]")); + assertTrue(traces.contains("trace4-1, trace4-2")); + } + + @Test + public void testNullTracing() { + Query q = new Query("?query=foo&traceLevel=2"); + assertEquals(2, q.getTraceLevel()); + q.trace(false,2, "trace2 ", null); + Set<String> traces = new HashSet<>(); + for (String trace : q.getContext(true).getTrace().traceNode().descendants(String.class)) { + traces.add(trace); + } + assertTrue(traces.contains("trace2 null")); + } + + @Test + public void testQueryPropertyResolveTracing() { + QueryProfile testProfile=new QueryProfile("test"); + testProfile.setOverridable("u", false, null); + testProfile.set("d","e", null); + testProfile.set("u","11", null); + testProfile.set("foo.bar", "wiz", null); + Query q = new Query(QueryTestCase.httpEncode("?query=a:>5&a=b&traceLevel=5&sources=a,b&u=12&foo.bar2=wiz2&c.d=foo&queryProfile=test"),testProfile.compile(null)); + String trace=q.getContext(false).getTrace().toString(); + String[] traceLines=trace.split("\n"); + for (String line : traceLines) + System.out.println(line); + assertTrue(contains("query=a:>5 (value from request)",traceLines)); + assertTrue(contains("traceLevel=5 (value from request)",traceLines)); + assertTrue(contains("a=b (value from request)",traceLines)); + assertTrue(contains("sources=[a, b] (value from request)",traceLines)); + assertTrue(contains("d=e (value from query profile)",traceLines)); + assertTrue(contains("u=11 (value from query profile - unoverridable, ignoring request value)",traceLines)); + } + + @Test + public void testNonleafInRequestDoesNotOverrideProfile() { + QueryProfile testProfile=new QueryProfile("test"); + testProfile.set("a.b", "foo", (QueryProfileRegistry)null); + testProfile.freeze(); + { + Query q = new Query("?", testProfile.compile(null)); + assertEquals("foo", q.properties().get("a.b")); + } + + { + Query q = new Query("?a=bar", testProfile.compile(null)); + assertEquals("bar", q.properties().get("a")); + assertEquals("foo", q.properties().get("a.b")); + } + } + + @Test + public void testQueryPropertyResolveTracing2() { + QueryProfile defaultProfile=new QueryProfile("default"); + defaultProfile.freeze(); + Query q = new Query(QueryTestCase.httpEncode("?query=dvd&a.b=foo&tracelevel=9"), defaultProfile.compile(null)); + String trace=q.getContext(false).getTrace().toString(); + String[] traceLines=trace.split("\n"); + assertTrue(contains("query=dvd (value from request)",traceLines)); + assertTrue(contains("a.b=foo (value from request)",traceLines)); + } + + @Test + public void testQueryPropertyListingAndTrace() { + QueryProfile defaultProfile=new QueryProfile("default"); + defaultProfile.setDimensions(new String[]{"x"}); + defaultProfile.set("a.b","a.b-x1-value",new String[] {"x1"}, null); + defaultProfile.set("a.b", "a.b-x2-value", new String[]{"x2"}, null); + defaultProfile.freeze(); + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map<String,Object> propertyList=q.properties().listProperties(); + assertEquals(3,propertyList.size()); + assertEquals("a.b-x1-value",propertyList.get("a.b")); + String trace=q.getContext(false).getTrace().toString(); + String[] traceLines=trace.split("\n"); + assertTrue(contains("a.b=a.b-x1-value (value from query profile)",traceLines)); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map<String,Object> propertyList=q.properties().listProperties("a"); + assertEquals(1,propertyList.size()); + assertEquals("a.b-x1-value",propertyList.get("b")); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x2"),defaultProfile.compile(null)); + Map<String,Object> propertyList=q.properties().listProperties(); + assertEquals(3,propertyList.size()); + assertEquals("a.b-x2-value",propertyList.get("a.b")); + String trace=q.getContext(false).getTrace().toString(); + String[] traceLines=trace.split("\n"); + assertTrue(contains("a.b=a.b-x2-value (value from query profile)",traceLines)); + } + } + + @Test + public void testQueryPropertyListingThreeLevel() { + QueryProfile defaultProfile=new QueryProfile("default"); + defaultProfile.setDimensions(new String[] {"x"}); + defaultProfile.set("a.b.c", "a.b.c-x1-value", new String[]{"x1"}, null); + defaultProfile.set("a.b.c", "a.b.c-x2-value", new String[]{"x2"}, null); + defaultProfile.freeze(); + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map<String,Object> propertyList=q.properties().listProperties(); + assertEquals(3,propertyList.size()); + assertEquals("a.b.c-x1-value",propertyList.get("a.b.c")); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map<String,Object> propertyList=q.properties().listProperties("a"); + assertEquals(1,propertyList.size()); + assertEquals("a.b.c-x1-value",propertyList.get("b.c")); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x1"),defaultProfile.compile(null)); + Map<String,Object> propertyList=q.properties().listProperties("a.b"); + assertEquals(1,propertyList.size()); + assertEquals("a.b.c-x1-value",propertyList.get("c")); + } + + { + Query q = new Query(QueryTestCase.httpEncode("?tracelevel=9&x=x2"),defaultProfile.compile(null)); + Map<String,Object> propertyList=q.properties().listProperties(); + assertEquals(3,propertyList.size()); + assertEquals("a.b.c-x2-value",propertyList.get("a.b.c")); + } + } + + @Test + public void testQueryPropertyReplacement() { + QueryProfile defaultProfile=new QueryProfile("default"); + defaultProfile.set("model.queryString","myquery", (QueryProfileRegistry)null); + defaultProfile.set("queryUrl","http://provider:80?query=%{model.queryString}", (QueryProfileRegistry)null); + defaultProfile.freeze(); + + Query q1 = new Query(QueryTestCase.httpEncode(""),defaultProfile.compile(null)); + assertEquals("myquery",q1.getModel().getQueryString()); + assertEquals("http://provider:80?query=myquery",q1.properties().get("queryUrl")); + + Query q2 = new Query(QueryTestCase.httpEncode("?model.queryString=foo"),defaultProfile.compile(null)); + assertEquals("foo",q2.getModel().getQueryString()); + assertEquals("http://provider:80?query=foo",q2.properties().get("queryUrl")); + + Query q3 = new Query(QueryTestCase.httpEncode("?query=foo"),defaultProfile.compile(null)); + assertEquals("foo",q3.getModel().getQueryString()); + assertEquals("http://provider:80?query=foo",q3.properties().get("queryUrl")); + + Query q4 = new Query(QueryTestCase.httpEncode("?query=foo"),defaultProfile.compile(null)); + q4.getModel().setQueryString("bar"); + assertEquals("http://provider:80?query=bar",q4.properties().get("queryUrl")); + } + + @Test + public void testNoQueryString() throws IOException { + Query q = new Query(httpEncode("?tracelevel=1")); + Chain<Searcher> chain = new Chain<>(new RandomSearcher()); + new Execution(chain, Execution.Context.createContextStub()).search(q); + assertNotNull(q.getModel().getQueryString()); + } + + @Test + public void testSetCollapseField() { + Query q = new Query(httpEncode("?collapsefield=foo&presentation.format=tiled")); + assertEquals("foo",q.properties().get("collapsefield")); + assertEquals("tiled", q.properties().get("presentation.format")); + assertEquals("tiled", q.getPresentation().getFormat()); + } + + @Test + public void testSetNullProperty() { + QueryProfile profile = new QueryProfile("test"); + profile.set("property","initialValue", (QueryProfileRegistry)null); + Query query = new Query(httpEncode("?query=test"), profile.compile(null)); + assertEquals("initialValue",query.properties().get("property")); + query.properties().set("property",null); + assertNull(query.properties().get("property")); + } + + @Test + public void testSetNullPropertyNoQueryProfile() { + Query query=new Query(); + query.properties().set("a",null); + assertNull(query.properties().get("a")); + } + + @Test + public void testMissingParameter() { + Query q=new Query("?query=foo&hits="); + assertEquals(0, q.errors().size()); + } + + @Test + public void testModelProperties() { + { + Query query=new Query(); + query.properties().set("model.searchPath", "foo"); + assertEquals("Set dynamic get dynamic works","foo",query.properties().get("model.searchPath")); + assertEquals("Set dynamic get static works","foo",query.getModel().getSearchPath()); + } + + { + Query query=new Query(); + query.getModel().setSearchPath("foo"); + assertEquals("Set static get dynamic works","foo",query.properties().get("model.searchPath")); + assertEquals("Set static get static works","foo",query.getModel().getSearchPath()); + } + + { + Query query=new Query(); + query.properties().set("a","bar"); + assertEquals("bar",query.properties().get("a")); + query.properties().set("a.b","baz"); + assertEquals("baz",query.properties().get("a.b")); + } + } + + @Test + public void testPositiveTerms() { + Query q = new Query(QueryTestCase.httpEncode("/?query=-a \"b c\" d e")); + Item i = q.getModel().getQueryTree().getRoot(); + List<IndexedItem> l = QueryTree.getPositiveTerms(i); + assertEquals(3, l.size()); + } + + protected boolean contains(String lineSubstring,String[] lines) { + for (String line : lines) + if (line.indexOf(lineSubstring)>=0) return true; + return false; + } + + private static class RandomSearcher extends Searcher { + + @Override + public Result search(Query query, Execution execution) { + Result r=new Result(query); + r.hits().add(new Hit("hello")); + return r; + } + } + + /** + * Url encode the given string, except the characters =?&, such that queries with paths and parameters can + * be written as a single string. + */ + public static String httpEncode(String s) { + try { + if (s == null) return null; + String encoded = URLEncoder.encode(s, "utf-8"); + encoded = encoded.replaceAll("%3F", "?"); + encoded = encoded.replaceAll("%3D", "="); + encoded = encoded.replaceAll("%26", "&"); + return encoded; + } + catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/test/RequestParameterPreservationTestCase.java b/container-search/src/test/java/com/yahoo/search/test/RequestParameterPreservationTestCase.java new file mode 100644 index 00000000000..5c77dc0215d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/test/RequestParameterPreservationTestCase.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.test; + +import com.yahoo.search.Query; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class RequestParameterPreservationTestCase extends junit.framework.TestCase { + + public void testPreservation() { + Query query=new Query("?query=test...&offset=15&hits=10"); + query.setWindow(25,13); + assertEquals(25,query.getOffset()); + assertEquals(13,query.getHits()); + assertEquals("15", query.getHttpRequest().getProperty("offset")); + assertEquals("10", query.getHttpRequest().getProperty("hits")); + assertEquals("test...",query.getHttpRequest().getProperty("query")); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/test/ResultBenchmark.java b/container-search/src/test/java/com/yahoo/search/test/ResultBenchmark.java new file mode 100644 index 00000000000..450da35b7a4 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/test/ResultBenchmark.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.test; + +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +/** + * Tests the speed of accessing hits in the query by id + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ResultBenchmark { + + public void run() { + int foundCount=0; + + // Warm-up + out("Warming up..."); + Result result=createResult(); + for (int i=0; i<10*1000; i++) + foundCount+=accessResultFiveTimes(result); + foundCount=0; + + long startTime=System.currentTimeMillis(); + out("Running..."); + for (int i=0; i<200*1000; i++) + foundCount+=accessResultFiveTimes(result); + out("Successfully looked up " + foundCount + " hits"); + long endTime=System.currentTimeMillis(); + out("Accessing a result 1.000.000 times took " + (endTime-startTime) + " ms"); + } + + private final Result createResult() { + // 8 sets, 8 gets + Result result=new Result(new Query("?query=test&hits=10&presentation.bolding=true&model.type=all")); + addHits(5,"firstTopLevel",result.hits()); + result.hits().add(addHits(10, "group1hit", new HitGroup())); + addHits(5, "secondTopLevel", result.hits()); + result.hits().add(addHits(10, "group2hit", new HitGroup())); + result.hits().add(addHits(10, "group3hit", new HitGroup())); + return result; + } + + private final HitGroup addHits(int count,String idPrefix,HitGroup to) { + for (int i=1; i<=count; i++) + to.add(new Hit(idPrefix + i,1/i)); + return to; + } + + private final int accessResultFiveTimes(Result result) { + // 8 sets, 8 gets + int foundCount=0; + if (null!=result.hits().get("firstTopLevel1")) + foundCount++; + if (null!=result.hits().get("secondTopLevel3")) + foundCount++; + if (null!=result.hits().get("group3hit5")) + foundCount++; + if (null!=result.hits().get("group1hit2")) + foundCount++; + if (null!=result.hits().get("group2hit4")) + foundCount++; + return foundCount; + } + + private void out(String string) { + System.out.println(string); + } + + public static void main(String[] args) { + new ResultBenchmark().run(); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java new file mode 100644 index 00000000000..ac6ceeb5467 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/FieldFilterTestCase.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.*; + + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; + +/** + * Smoketest that we remove fields in a sane manner. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class FieldFilterTestCase { + private static final String FIELD_C = "c"; + private static final String FIELD_B = "b"; + private static final String FIELD_A = "a"; + private Chain<Searcher> searchChain; + private Execution.Context context; + private Execution execution; + + @Before + public void setUp() throws Exception { + Query query = new Query("?query=test"); + + Result result = new Result(query); + Hit hit = createHit("lastHit", .1d, FIELD_A, FIELD_B, FIELD_C); + result.hits().add(hit); + + DocumentSourceSearcher mockBackend = new DocumentSourceSearcher(); + mockBackend.addResult(query, result); + + searchChain = new Chain<Searcher>(new FieldFilter(), + mockBackend); + context = Execution.Context.createContextStub(null); + execution = new Execution(searchChain, context); + + } + + private Hit createHit(String id, double relevancy, String... fieldNames) { + Hit h = new Hit(id, relevancy); + h.setFillable(); + int i = 0; + for (String field : fieldNames) { + h.setField(field, ++i); + } + return h; + } + + @After + public void tearDown() throws Exception { + searchChain = null; + context = null; + execution = null; + } + + @Test + public final void testBasic() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD_B); + Result result = execution.search(query); + execution.fill(result); + assertEquals(1, result.getConcreteHitCount()); + assertFalse(result.hits().get(0).fieldKeys().contains(FIELD_A)); + assertTrue(result.hits().get(0).fieldKeys().contains(FIELD_B)); + assertFalse(result.hits().get(0).fieldKeys().contains(FIELD_C)); + } + + @Test + public final void testNoFiltering() { + final Query query = new Query("?query=test"); + Result result = execution.search(query); + execution.fill(result); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).fieldKeys().contains(FIELD_A)); + assertTrue(result.hits().get(0).fieldKeys().contains(FIELD_B)); + assertTrue(result.hits().get(0).fieldKeys().contains(FIELD_C)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java new file mode 100644 index 00000000000..7834539db72 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/MinimalQueryInserterTestCase.java @@ -0,0 +1,297 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.*; + +import com.yahoo.search.grouping.GroupingRequest; + +import org.apache.http.client.utils.URIBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import com.yahoo.collections.Tuple2; +import com.yahoo.component.Version; +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.query.Sorting.AttributeSorter; +import com.yahoo.search.query.Sorting.FieldOrder; +import com.yahoo.search.query.Sorting.LowerCaseSorter; +import com.yahoo.search.query.Sorting.Order; +import com.yahoo.search.query.Sorting.UcaSorter; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; + +import java.util.ArrayList; +import java.util.List; + +/** + * Smoke test for first generation YQL+ integration. + */ +public class MinimalQueryInserterTestCase { + private Chain<Searcher> searchChain; + private Execution.Context context; + private Execution execution; + + @Before + public void setUp() throws Exception { + searchChain = new Chain<Searcher>(new MinimalQueryInserter()); + context = Execution.Context.createContextStub(null); + execution = new Execution(searchChain, context); + } + + @After + public void tearDown() throws Exception { + searchChain = null; + context = null; + execution = null; + } + + @Test + public void requireThatGroupingStepsAreAttachedToQuery() { + URIBuilder builder = new URIBuilder(); + builder.setPath("search/"); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox';"); + Query query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[]", query); + + assertEquals(1, query.getPresentation().getSummaryFields().size()); + assertEquals("foo", query.getPresentation().getSummaryFields().toArray(new String[1])[0]); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + + "| all(group(a) each(output(count())));"); + query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[[]all(group(a) each(output(count())))]", query); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + + "| all(group(a) each(output(count()))) " + + "| all(group(b) each(output(count())));"); + query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[[]all(group(a) each(output(count())))," + + " []all(group(b) each(output(count())))]", query); + } + + @Test + public void requireThatGroupingContinuationsAreAttachedToQuery() { + URIBuilder builder = new URIBuilder(); + builder.setPath("search/"); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox';"); + Query query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[]", query); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + + "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "all(group(a) each(output(count())));"); + query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))]", query); + + builder.setParameter("yql", "select foo from bar where baz contains 'cox' " + + "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "all(group(a) each(output(count()))) " + + "| [{ 'continuations':['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]" + + "all(group(b) each(output(count())));"); + query = new Query(builder.toString()); + execution.search(query); + assertEquals("baz:cox", query.getModel().getQueryTree().toString()); + assertGrouping("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))," + + " [BCBBBBBDBF, BCBJBPCBJCCJ]all(group(b) each(output(count())))]", query); + } + + @Test + @Ignore + // TODO: YQL work in progress (jon) + public final void testTmp() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + //execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + } + + @Test + public final void testSearch() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + } + + @Test + public final void testUserQueryFailsWithoutArgument() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + } + + @Test + public final void testSearchFromAllSourcesWithUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(0, query.getModel().getSources().size()); + } + + @Test + public final void testSearchFromAllSourcesWithoutUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(0, query.getModel().getSources().size()); + } + + @Test + public final void testSearchFromSomeSourcesWithoutUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(2, query.getModel().getSources().size()); + assertTrue(query.getModel().getSources().contains("sourceA")); + assertTrue(query.getModel().getSources().contains("sourceB")); + } + + @Test + public final void testSearchFromSomeSourcesWithUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&sources=abc&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(3, query.getModel().getSources().size()); + assertTrue(query.getModel().getSources().contains("sourceA")); + assertTrue(query.getModel().getSources().contains("sourceB")); + assertTrue(query.getModel().getSources().contains("abc")); + } + + @Test + public final void testSearchFromSomeSourcesWithOverlappingUserSource() { + final Query query = new Query("search/?query=easilyRecognizedString&sources=abc,sourceA&yql=select%20ignoredfield%20from%20sources%20sourceA,%20sourceB%20where%20title%20contains%20%22madonna%22%20and%20userQuery()%3B"); + execution.search(query); + assertEquals("AND title:madonna easilyRecognizedString", query.getModel().getQueryTree().toString()); + assertEquals(3, query.getModel().getSources().size()); + assertTrue(query.getModel().getSources().contains("sourceA")); + assertTrue(query.getModel().getSources().contains("sourceB")); + assertTrue(query.getModel().getSources().contains("abc")); + } + + @Test + public final void testLimitAndOffset() { + final Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2031offset%207%3B"); + execution.search(query); + assertEquals(7, query.getOffset()); + assertEquals(24, query.getHits()); + assertEquals("select * from sources * where title contains \"madonna\" limit 31 offset 7;", + query.yqlRepresentation()); + } + + @Test + public final void testMaxOffset() { + final Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040031offset%2040000%3B"); + Result r = execution.search(query); + assertEquals(1, r.hits().getErrorHit().errors().size()); + ErrorMessage e = r.hits().getErrorHit().errorIterator().next(); + assertEquals(com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code, e.getCode()); + assertTrue(e.getDetailedMessage().indexOf("max offset") >= 0); + } + + @Test + public final void testMaxLimit() { + final Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20limit%2040000offset%207%3B"); + Result r = execution.search(query); + assertEquals(1, r.hits().getErrorHit().errors().size()); + ErrorMessage e = r.hits().getErrorHit().errorIterator().next(); + assertEquals(com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER.code, e.getCode()); + assertTrue(e.getDetailedMessage().indexOf("max hits") >= 0); + } + + @Test + public final void testTimeout() { + final Query query = new Query("search/?yql=select%20*%20from%20sources%20*%20where%20title%20contains%20%22madonna%22%20timeout%2051%3B"); + execution.search(query); + assertEquals(51L, query.getTimeout()); + assertEquals("select * from sources * where title contains \"madonna\" timeout 51;", query.yqlRepresentation()); + } + + @Test + public final void testOrdering() { + { + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203%3B"; + Query query = new Query("search/?yql=" + yql); + execution.search(query); + assertEquals(2, query.getRanking().getSorting().fieldOrders() + .size()); + assertEquals("something", query.getRanking().getSorting() + .fieldOrders().get(0).getFieldName()); + assertEquals(Order.ASCENDING, query.getRanking().getSorting() + .fieldOrders().get(0).getSortOrder()); + assertEquals("shoesize", query.getRanking().getSorting() + .fieldOrders().get(1).getFieldName()); + assertEquals(Order.DESCENDING, query.getRanking().getSorting() + .fieldOrders().get(1).getSortOrder()); + assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by something, shoesize desc limit 300 timeout 3;", query.yqlRepresentation()); + } + { + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20other%20limit%20300%20timeout%203%3B"; + Query query = new Query("search/?yql=" + yql); + execution.search(query); + assertEquals("other", query.getRanking().getSorting().fieldOrders() + .get(0).getFieldName()); + assertEquals(Order.ASCENDING, query.getRanking().getSorting() + .fieldOrders().get(0).getSortOrder()); + assertEquals("select ignoredfield from ignoredsource where title contains \"madonna\" order by other limit 300 timeout 3;", query.yqlRepresentation()); + } + { + String yql = "select%20foo%20from%20bar%20where%20title%20contains%20%22madonna%22%20order%20by%20%5B%7B%22function%22%3A%20%22uca%22%2C%20%22locale%22%3A%20%22en_US%22%2C%20%22strength%22%3A%20%22IDENTICAL%22%7D%5Dother%20desc%2C%20%5B%7B%22function%22%3A%20%22lowercase%22%7D%5Dsomething%20limit%20300%20timeout%203%3B"; + Query query = new Query("search/?yql=" + yql); + execution.search(query); + { + final FieldOrder fieldOrder = query.getRanking().getSorting() + .fieldOrders().get(0); + assertEquals("other", fieldOrder.getFieldName()); + assertEquals(Order.DESCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(UcaSorter.class, sorter.getClass()); + final UcaSorter uca = (UcaSorter) sorter; + assertEquals("en_US", uca.getLocale()); + assertEquals(UcaSorter.Strength.IDENTICAL, uca.getStrength()); + } + { + final FieldOrder fieldOrder = query.getRanking().getSorting() + .fieldOrders().get(1); + assertEquals("something", fieldOrder.getFieldName()); + assertEquals(Order.ASCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(LowerCaseSorter.class, sorter.getClass()); + } + assertEquals("select foo from bar where title contains \"madonna\" order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc, [{\"function\": \"lowercase\"}]something limit 300 timeout 3;", + query.yqlRepresentation()); + } + } + + @Test + public final void testStringReprBasicSanity() { + String yql = "select%20ignoredfield%20from%20ignoredsource%20where%20title%20contains%20%22madonna%22%20order%20by%20something%2C%20shoesize%20desc%20limit%20300%20timeout%203%3B"; + Query query = new Query("search/?yql=" + yql); + execution.search(query); + assertEquals("select ignoredfield from ignoredsource where [{\"segmenter\": {\"version\": \"1.9\", \"backend\": \"YqlUnitTest\"}}](title contains \"madonna\") order by something, shoesize desc limit 300 timeout 3;", + query.yqlRepresentation(new Tuple2<>("YqlUnitTest", new Version(1, 9)), true)); + } + + + private static void assertGrouping(String expected, Query query) { + List<String> actual = new ArrayList<>(); + for (GroupingRequest request : GroupingRequest.getRequests(query)) { + actual.add(request.continuations().toString() + request.getRootOperation()); + } + assertEquals(expected, actual.toString()); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java new file mode 100644 index 00000000000..8c4d8e0fe84 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/ResegmentingTestCase.java @@ -0,0 +1,147 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.assertEquals; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; + +/** + * Check rules for resegmenting words in YQL+ when segmenter is deemed + * incompatible. The class under testing is {@link YqlParser}. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class ResegmentingTestCase { + private YqlParser parser; + + @Before + public void setUp() throws Exception { + ParserEnvironment env = new ParserEnvironment(); + parser = new YqlParser(env); + } + + @After + public void tearDown() throws Exception { + parser = null; + } + + @Test + public final void testWord() { + assertEquals( + "title:'a b'", + parser.parse( + new Parsable() + .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}] (title contains \"a b\");")) + .toString()); + } + + @Test + public final void testPhraseSegment() { + assertEquals( + "title:'c d'", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}}]" + + " phrase(\"a\", \"b\")));")) + .toString()); + } + + @Test + public final void testPhraseInEquiv() { + assertEquals( + "EQUIV title:a title:'c d'", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains" + + " equiv(\"a\"," + + " ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}}]\"b\")" + + ")" + + ");")) + .toString()); + } + + @Test + public final void testPhraseSegmentToAndSegment() { + assertEquals( + "SAND title:c title:d", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"andSegmenting\": true}]" + + " phrase(\"a\", \"b\")));")) + .toString()); + } + + @Test + public final void testPhraseSegmentInPhrase() { + assertEquals( + "title:\"a 'c d'\"", + parser.parse( + new Parsable() + .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains phrase(\"a\"," + + " ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}}]" + + " phrase(\"e\", \"f\"))));")) + .toString()); + } + + @Test + public final void testWordNoImplicitTransforms() { + assertEquals( + "title:a b", + parser.parse( + new Parsable() + .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}] (title contains ([{\"implicitTransforms\": false}]\"a b\"));")) + .toString()); + } + + @Test + public final void testPhraseSegmentNoImplicitTransforms() { + assertEquals( + "title:'a b'", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"implicitTransforms\": false}]" + + " phrase(\"a\", \"b\")));")) + .toString()); + } + + @Test + public final void testPhraseSegmentToAndSegmentNoImplicitTransforms() { + assertEquals( + "SAND title:a title:b", + parser.parse( + new Parsable() + .setQuery("select * from sources * where" + + " [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"andSegmenting\": true, \"implicitTransforms\": false}]" + + " phrase(\"a\", \"b\")));")) + .toString()); + } + + @Test + public final void testPhraseSegmentInPhraseNoImplicitTransforms() { + assertEquals( + "title:\"a 'e f'\"", + parser.parse( + new Parsable() + .setQuery("select * from sources * where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": \"nonexistant\"}}]" + + " (title contains phrase(\"a\"," + + " ([{\"origin\": {\"offset\": 0, \"length\":3, \"original\": \"c d\"}, \"implicitTransforms\": false}]" + + " phrase(\"e\", \"f\"))));")) + .toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java new file mode 100644 index 00000000000..0d81970bdce --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -0,0 +1,280 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.*; + +import org.apache.http.client.utils.URIBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; + +import static com.yahoo.container.protect.Error.INVALID_QUERY_PARAMETER; + +/** + * Tests where you really test YqlParser but need the full Query infrastructure. + * + * @author steinar + */ +public class UserInputTestCase { + + private Chain<Searcher> searchChain; + private Execution.Context context; + private Execution execution; + + @Before + public void setUp() throws Exception { + searchChain = new Chain<Searcher>(new MinimalQueryInserter()); + context = Execution.Context.createContextStub(null); + execution = new Execution(searchChain, context); + } + + @After + public void tearDown() throws Exception { + searchChain = null; + context = null; + execution = null; + } + + @Test + public final void testSimpleUserInput() { + { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains \"nalle\";", query.yqlRepresentation()); + } + { + URIBuilder builder = searchUri(); + builder.setParameter("nalle", "bamse"); + builder.setParameter("yql", + "select * from sources * where userInput(@nalle);"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains \"bamse\";", query.yqlRepresentation()); + } + { + URIBuilder builder = searchUri(); + builder.setParameter("nalle", "bamse"); + builder.setParameter("yql", + "select * from sources * where userInput(nalle);"); + Query query = new Query(builder.toString()); + Result r = execution.search(query); + assertNotNull(r.hits().getError()); + } + } + + @Test + public final void testRawUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"grammar\": \"raw\"}]userInput(\"nal le\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains \"nal le\";", query.yqlRepresentation()); + } + + @Test + public final void testSegmentedUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"nal le\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains ([{\"origin\": {\"original\": \"nal le\", \"offset\": 0, \"length\": 6}}]phrase(\"nal\", \"le\"));", query.yqlRepresentation()); + } + + @Test + public final void testSegmentedNoiseUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"grammar\": \"segment\"}]userInput(\"^^^^^^^^\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where default contains \"^^^^^^^^\";", query.yqlRepresentation()); + } + + @Test + public final void testCustomDefaultIndexUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"defaultIndex\": \"glompf\"}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where glompf contains \"nalle\";", query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputStemming() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"stem\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"stem\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputUnrankedTerms() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"ranked\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"ranked\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputFiltersTerms() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"filter\": true}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"filter\": true}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputCaseNormalization() { + URIBuilder builder = searchUri(); + builder.setParameter( + "yql", + "select * from sources * where [{\"normalizeCase\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"normalizeCase\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputAccentRemoval() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"accentDrop\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"accentDrop\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testAnnotatedUserInputPositionData() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where [{\"usePositionData\": false}]userInput(\"nalle\");"); + Query query = searchAndAssertNoErrors(builder); + assertEquals( + "select * from sources * where default contains ([{\"usePositionData\": false}]\"nalle\");", + query.yqlRepresentation()); + } + + @Test + public final void testQueryPropertiesAsStringArguments() { + URIBuilder builder = searchUri(); + builder.setParameter("nalle", "bamse"); + builder.setParameter("meta", "syntactic"); + builder.setParameter("yql", + "select * from sources * where foo contains @nalle and foo contains phrase(@nalle, @meta, @nalle);"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where (foo contains \"bamse\" AND foo contains phrase(\"bamse\", \"syntactic\", \"bamse\"));", query.yqlRepresentation()); + } + + private Query searchAndAssertNoErrors(URIBuilder builder) { + Query query = new Query(builder.toString()); + Result r = execution.search(query); + assertNull(r.hits().getError()); + return query; + } + + private URIBuilder searchUri() { + URIBuilder builder = new URIBuilder(); + builder.setPath("search/"); + return builder; + } + + @Test + public final void testEmptyUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where userInput(\"\");"); + assertQueryFails(builder); + } + + @Test + public final void testEmptyUserInputFromQueryProperty() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ""); + builder.setParameter("yql", + "select * from sources * where userInput(@foo);"); + assertQueryFails(builder); + } + + @Test + public final void testEmptyQueryProperty() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ""); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains @foo);"); + assertQueryFails(builder); + } + + @Test + public final void testEmptyQueryPropertyInsideExpression() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ""); + builder.setParameter("yql", + "select * from sources * where bar contains \"a\" and nonEmpty(bar contains \"bar\" and foo contains @foo);"); + assertQueryFails(builder); + } + + @Test + public final void testCompositeWithoutArguments() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and foo contains phrase();"); + searchAndAssertNoErrors(builder); + builder = searchUri(); + builder.setParameter("yql", "select * from sources * where bar contains \"a\" and nonEmpty(foo contains phrase());"); + assertQueryFails(builder); + } + + @Test + public final void testAnnoyingPlacementOfNonEmpty() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", + "select * from sources * where bar contains \"a\" and foo contains nonEmpty(phrase(\"a\", \"b\"));"); + assertQueryFails(builder); + } + + private void assertQueryFails(URIBuilder builder) { + Result r = execution.search(new Query(builder.toString())); + assertEquals(INVALID_QUERY_PARAMETER.code, r.hits().getError().getCode()); + } + + @Test + public final void testAllowEmptyUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ""); + builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo);"); + searchAndAssertNoErrors(builder); + } + + @Test + public final void testAllowEmptyNullFromQueryParsing() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ",,,,,,,,"); + builder.setParameter("yql", "select * from sources * where [{\"allowEmpty\": true}]userInput(@foo);"); + searchAndAssertNoErrors(builder); + } + + @Test + public final void testDisallowEmptyNullFromQueryParsing() { + URIBuilder builder = searchUri(); + builder.setParameter("foo", ",,,,,,,,"); + builder.setParameter("yql", "select * from sources * where userInput(@foo);"); + assertQueryFails(builder); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java new file mode 100644 index 00000000000..d9d8eb1b14b --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -0,0 +1,404 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.*; + +import com.yahoo.search.Query; +import com.yahoo.search.grouping.Continuation; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.grouping.request.AllOperation; +import com.yahoo.search.grouping.request.AttributeFunction; +import com.yahoo.search.grouping.request.CountAggregator; +import com.yahoo.search.grouping.request.EachOperation; +import com.yahoo.search.grouping.request.GroupingOperation; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.MarkerWordItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; + +import java.util.Arrays; + +public class VespaSerializerTestCase { + + private static final String SELECT = "select ignoredfield from sourceA where "; + private YqlParser parser; + + @Before + public void setUp() throws Exception { + ParserEnvironment env = new ParserEnvironment(); + parser = new YqlParser(env); + } + + @After + public void tearDown() throws Exception { + parser = null; + } + + @Test + public void requireThatGroupingRequestsAreSerialized() { + Query query = new Query(); + query.getModel().getQueryTree().setRoot(new WordItem("foo")); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\")", + VespaSerializer.serialize(query)); + + newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("a")) + .addChild(new EachOperation().addOutput(new CountAggregator()))); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + "| all(group(attribute(a)) each(output(count())))", + VespaSerializer.serialize(query)); + + newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("b")) + .addChild(new EachOperation().addOutput(new CountAggregator()))); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + "| all(group(attribute(a)) each(output(count()))) " + + "| all(group(attribute(b)) each(output(count())))", + VespaSerializer.serialize(query)); + } + + @Test + public void requireThatGroupingContinuationsAreSerialized() { + Query query = new Query(); + query.getModel().getQueryTree().setRoot(new WordItem("foo")); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\")", + VespaSerializer.serialize(query)); + + newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("a")) + .addChild(new EachOperation().addOutput(new CountAggregator())), + Continuation.fromString("BCBCBCBEBG"), + Continuation.fromString("BCBKCBACBKCCK")); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "all(group(attribute(a)) each(output(count())))", + VespaSerializer.serialize(query)); + + newGroupingRequest(query, new AllOperation().setGroupBy(new AttributeFunction("b")) + .addChild(new EachOperation().addOutput(new CountAggregator())), + Continuation.fromString("BCBBBBBDBF"), + Continuation.fromString("BCBJBPCBJCCJ")); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"foo\") " + + "| [{ 'continuations':['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]" + + "all(group(attribute(a)) each(output(count()))) " + + "| [{ 'continuations':['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]" + + "all(group(attribute(b)) each(output(count())))", + VespaSerializer.serialize(query)); + } + + @Test + public final void testAnd() { + parseAndConfirm("(description contains \"a\" AND title contains \"that\")"); + } + + private void parseAndConfirm(String expected) { + parseAndConfirm(expected, expected); + } + + private void parseAndConfirm(String expected, String toParse) { + QueryTree item = parser + .parse(new Parsable() + .setQuery(SELECT + toParse + ";")); + // System.out.println(item.toString()); + String q = VespaSerializer.serialize(item.getRoot()); + assertEquals(expected, q); + } + + @Test + public final void testAndNot() { + parseAndConfirm("(description contains \"a\") AND !(title contains \"that\")"); + } + + @Test + public final void testEquiv() { + parseAndConfirm("title contains equiv(\"a\", \"b\")"); + } + + @Test + public final void testNear() { + parseAndConfirm("title contains near(\"a\", \"b\")"); + parseAndConfirm("title contains ([{\"distance\": 50}]near(\"a\", \"b\"))"); + } + + @Test + public final void testNumbers() { + parseAndConfirm("title = 500"); + parseAndConfirm("title > 500"); + parseAndConfirm("title < 500"); + } + + @Test + public final void testAnnotatedNumbers() { + parseAndConfirm("title = ([{\"filter\": true}]500)"); + parseAndConfirm("title > ([{\"filter\": true}]500)"); + parseAndConfirm("title < ([{\"filter\": true}](-500))"); + parseAndConfirm("title <= ([{\"filter\": true}](-500))", "([{\"filter\": true}](-500)) >= title"); + parseAndConfirm("title <= ([{\"filter\": true}](-500))"); + } + + @Test + public final void testRange() { + parseAndConfirm("range(title, 1, 500)"); + } + + @Test + public final void testAnnotatedRange() { + parseAndConfirm("[{\"filter\": true}]range(title, 1, 500)"); + } + + @Test + public final void testOrderedNear() { + parseAndConfirm("title contains onear(\"a\", \"b\")"); + } + + @Test + public final void testOr() { + parseAndConfirm("(description contains \"a\" OR title contains \"that\")"); + } + + @Test + public final void testDotProduct() { + parseAndConfirm("dotProduct(description, {\"a\": 1, \"b\": 2})"); + } + + @Test + public final void testPredicate() { + parseAndConfirm("predicate(boolean,{\"gender\":\"male\"},{\"age\":25L})"); + parseAndConfirm("predicate(boolean,{\"gender\":\"male\",\"hobby\":\"music\",\"hobby\":\"hiking\"}," + + "{\"age\":25L})", + "predicate(boolean,{\"gender\":\"male\",\"hobby\":[\"music\",\"hiking\"]},{\"age\":25})"); + parseAndConfirm("predicate(boolean,{\"0x3\":{\"gender\":\"male\"},\"0x1\":{\"hobby\":\"music\"},\"0x1\":{\"hobby\":\"hiking\"}},{\"0x80ffffffffffffff\":{\"age\":23L}})", + "predicate(boolean,{\"0x3\":{\"gender\":\"male\"},\"0x1\":{\"hobby\":[\"music\",\"hiking\"]}},{\"0x80ffffffffffffff\":{\"age\":23L}})"); + parseAndConfirm("predicate(boolean,0,0)"); + parseAndConfirm("predicate(boolean,0,0)","predicate(boolean,null,void)"); + parseAndConfirm("predicate(boolean,0,0)","predicate(boolean,{},{})"); + } + + @Test + public final void testPhrase() { + parseAndConfirm("description contains phrase(\"a\", \"b\")"); + } + + @Test + public final void testAnnotatedPhrase() { + parseAndConfirm("description contains ([{\"id\": 1}]phrase(\"a\", \"b\"))"); + } + + @Test + public final void testAnnotatedNear() { + parseAndConfirm("description contains ([{\"distance\": 37}]near(\"a\", \"b\"))"); + } + + @Test + public final void testAnnotatedOnear() { + parseAndConfirm("description contains ([{\"distance\": 37}]onear(\"a\", \"b\"))"); + } + + @Test + public final void testAnnotatedEquiv() { + parseAndConfirm("description contains ([{\"id\": 1}]equiv(\"a\", \"b\"))"); + } + + @Test + public final void testAnnotatedPhraseSegment() { + PhraseSegmentItem phraseSegment = new PhraseSegmentItem("abc", true, false); + phraseSegment.addItem(new WordItem("a", "indexNamePlaceholder")); + phraseSegment.addItem(new WordItem("b", "indexNamePlaceholder")); + phraseSegment.setIndexName("someIndexName"); + phraseSegment.setLabel("labeled"); + phraseSegment.lock(); + String q = VespaSerializer.serialize(phraseSegment); + assertEquals("someIndexName contains ([{\"origin\": {\"original\": \"abc\", \"offset\": 0, \"length\": 3}, \"label\": \"labeled\"}]phrase(\"a\", \"b\"))", q); + } + + @Test + public final void testAnnotatedAndSegment() { + AndSegmentItem andSegment = new AndSegmentItem("abc", true, false); + andSegment.addItem(new WordItem("a", "indexNamePlaceholder")); + andSegment.addItem(new WordItem("b", "indexNamePlaceholder")); + andSegment.setLabel("labeled"); + andSegment.lock(); + String q = VespaSerializer.serialize(andSegment); + assertEquals("indexNamePlaceholder contains ([{\"origin\": {\"original\": \"abc\", \"offset\": 0, \"length\": 3}, \"andSegmenting\": true}]phrase(\"a\", \"b\"))", q); + } + + @Test + public final void testPhraseWithAnnotations() { + parseAndConfirm("description contains phrase(([{\"id\": 15}]\"a\"), \"b\")"); + } + + @Test + public final void testPhraseSegmentInPhrase() { + parseAndConfirm("description contains phrase(\"a\", \"b\", ([{\"origin\": {\"original\": \"c d\", \"offset\": 0, \"length\": 3}}]phrase(\"c\", \"d\")))"); + } + + @Test + public final void testRank() { + parseAndConfirm("rank(a contains \"A\", b contains \"B\")"); + } + + @Test + public final void testWand() { + parseAndConfirm("wand(description, {\"a\": 1, \"b\": 2})"); + } + + @Test + public final void testWeakAnd() { + parseAndConfirm("weakAnd(a contains \"A\", b contains \"B\")"); + } + + @Test + public final void testAnnotatedWeakAnd() { + parseAndConfirm("([{\"" + YqlParser.TARGET_NUM_HITS + "\": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); + parseAndConfirm("([{\"" + YqlParser.SCORE_THRESHOLD + "\": 10}]weakAnd(a contains \"A\", b contains \"B\"))"); + parseAndConfirm("([{\"" + YqlParser.TARGET_NUM_HITS + "\": 10, \"" + YqlParser.SCORE_THRESHOLD + + "\": 20}]weakAnd(a contains \"A\", b contains \"B\"))"); + } + + @Test + public final void testWeightedSet() { + parseAndConfirm("weightedSet(description, {\"a\": 1, \"b\": 2})"); + } + + @Test + public final void testAnnotatedWord() { + parseAndConfirm("description contains ([{\"andSegmenting\": true}]\"a\")"); + parseAndConfirm("description contains ([{\"weight\": 37}]\"a\")"); + parseAndConfirm("description contains ([{\"id\": 37}]\"a\")"); + parseAndConfirm("description contains ([{\"filter\": true}]\"a\")"); + parseAndConfirm("description contains ([{\"ranked\": false}]\"a\")"); + parseAndConfirm("description contains ([{\"significance\": 37.0}]\"a\")"); + parseAndConfirm("description contains ([{\"implicitTransforms\": false}]\"a\")"); + parseAndConfirm("(description contains ([{\"connectivity\": {\"id\": 2, \"weight\": 0.42}, \"id\": 1}]\"a\") AND description contains ([{\"id\": 2}]\"b\"))"); + } + + @Test + public final void testPrefix() { + parseAndConfirm("description contains ([{\"prefix\": true}]\"a\")"); + } + + @Test + public final void testSuffix() { + parseAndConfirm("description contains ([{\"suffix\": true}]\"a\")"); + } + + @Test + public final void testSubstring() { + parseAndConfirm("description contains ([{\"substring\": true}]\"a\")"); + } + + @Test + public final void testExoticItemTypes() { + Item item = MarkerWordItem.createEndOfHost(); + String q = VespaSerializer.serialize(item); + assertEquals("default contains ([{\"implicitTransforms\": false}]\"$\")", q); + } + + @Test + public final void testEmptyIndex() { + Item item = new WordItem("nalle", true); + String q = VespaSerializer.serialize(item); + assertEquals("default contains \"nalle\"", q); + } + + @Test + public final void testLongAndNot() { + NotItem item = new NotItem(); + item.addItem(new WordItem("a")); + item.addItem(new WordItem("b")); + item.addItem(new WordItem("c")); + item.addItem(new WordItem("d")); + String q = VespaSerializer.serialize(item); + assertEquals("(default contains ([{\"implicitTransforms\": false}]\"a\")) AND !(default contains ([{\"implicitTransforms\": false}]\"b\") OR default contains ([{\"implicitTransforms\": false}]\"c\") OR default contains ([{\"implicitTransforms\": false}]\"d\"))", q); + } + + @Test + public final void testPhraseAsOperatorArgument() { + // flattening phrases is a feature, not a bug + parseAndConfirm("description contains phrase(\"a\", \"b\", \"c\")", + "description contains phrase(\"a\", phrase(\"b\", \"c\"))"); + parseAndConfirm("description contains equiv(\"a\", phrase(\"b\", \"c\"))"); + } + + private static void newGroupingRequest(Query query, GroupingOperation grouping, Continuation... continuations) { + GroupingRequest request = GroupingRequest.newInstance(query); + request.setRootOperation(grouping); + request.continuations().addAll(Arrays.asList(continuations)); + } + + @Test + public final void testNumberTypeInt() { + parseAndConfirm("title = 500"); + parseAndConfirm("title > 500"); + parseAndConfirm("title < (-500)"); + parseAndConfirm("title >= (-500)"); + parseAndConfirm("title <= (-500)"); + parseAndConfirm("range(title, 0, 500)"); + } + + @Test + public final void testNumberTypeLong() { + parseAndConfirm("title = 549755813888L"); + parseAndConfirm("title > 549755813888L"); + parseAndConfirm("title < (-549755813888L)"); + parseAndConfirm("title >= (-549755813888L)"); + parseAndConfirm("title <= (-549755813888L)"); + parseAndConfirm("range(title, -549755813888L, 549755813888L)"); + } + + @Test + public final void testNumberTypeFloat() { + parseAndConfirm("title = 500.0"); // silly + parseAndConfirm("title > 500.0"); + parseAndConfirm("title < (-500.0)"); + parseAndConfirm("title >= (-500.0)"); + parseAndConfirm("title <= (-500.0)"); + parseAndConfirm("range(title, 0.0, 500.0)"); + } + + @Test + public final void testAnnotatedLong() { + parseAndConfirm("title >= ([{\"id\": 2014}](-549755813888L))"); + } + + @Test + public final void testHitLimit() { + parseAndConfirm("title <= ([{\"hitLimit\": 89}](-500))"); + parseAndConfirm("title <= ([{\"hitLimit\": 89}](-500))"); + parseAndConfirm("[{\"hitLimit\": 89}]range(title, 1, 500)"); + } + + @Test + public final void testOpenIntervals() { + parseAndConfirm("range(title, 0.0, 500.0)"); + parseAndConfirm("[{\"bounds\": \"open\"}]range(title, 0.0, 500.0)"); + parseAndConfirm("[{\"bounds\": \"leftOpen\"}]range(title, 0.0, 500.0)"); + parseAndConfirm("[{\"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0)"); + parseAndConfirm("[{\"id\": 500, \"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0)"); + } + + @Test + public final void testRegExp() { + parseAndConfirm("foo matches \"a b\""); + } + + @Test + public final void testWordAlternatives() { + parseAndConfirm("foo contains" + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + + "alternatives({\"trees\": 1.0, \"tree\": 0.7}))"); + } + + @Test + public final void testWordAlternativesInPhrase() { + parseAndConfirm("foo contains phrase(\"forest\"," + + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + + "alternatives({\"trees\": 1.0, \"tree\": 0.7}))" + + ")"); + } +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java new file mode 100644 index 00000000000..2ba5a781ab5 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass.Fields; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; +import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS;; + +/** + * Test translation of fields and sources in YQL+ to the associated concepts in + * Vespa. + */ +public class YqlFieldAndSourceTestCase { + private static final String FIELD1 = "field1"; + private static final String FIELD2 = "field2"; + private static final String FIELD3 = "field3"; + private static final String THIRD_OPTION = "THIRD_OPTION"; + + private Chain<Searcher> searchChain; + private Execution.Context context; + private Execution execution; + + + @Before + public void setUp() throws Exception { + Query query = new Query("?query=test"); + + Result result = new Result(query); + Hit hit = createHit("lastHit", .1d, FIELD1, FIELD2, FIELD3); + result.hits().add(hit); + + DocumentSourceSearcher mockBackend = new DocumentSourceSearcher(); + mockBackend.addResult(query, result); + + mockBackend.addSummaryClassByCopy(DEFAULT_SUMMARY_CLASS, Arrays.asList(FIELD1, FIELD2)); + mockBackend.addSummaryClassByCopy(Execution.ATTRIBUTEPREFETCH, Arrays.asList(FIELD2)); + mockBackend.addSummaryClassByCopy(THIRD_OPTION, Arrays.asList(FIELD3)); + + DocumentdbInfoConfig config = new DocumentdbInfoConfig( + new DocumentdbInfoConfig.Builder() + .documentdb(buildDocumentdbArray())); + + searchChain = new Chain<Searcher>(new FieldFiller(config), + mockBackend); + context = Execution.Context.createContextStub(null); + execution = new Execution(searchChain, context); + } + + private Hit createHit(String id, double relevancy, String... fieldNames) { + Hit h = new Hit(id, relevancy); + h.setFillable(); + int i = 0; + for (String field : fieldNames) { + h.setField(field, ++i); + } + return h; + } + + private List<Documentdb.Builder> buildDocumentdbArray() { + List<Documentdb.Builder> configArray = new ArrayList<Documentdb.Builder>( + 1); + configArray.add(new Documentdb.Builder().summaryclass( + buildSummaryclassArray()).name("defaultsearchdefinition")); + + return configArray; + } + + private List<Summaryclass.Builder> buildSummaryclassArray() { + return Arrays.asList( + new Summaryclass.Builder() + .id(0) + .name(DEFAULT_SUMMARY_CLASS) + .fields(Arrays.asList(new Fields.Builder().name(FIELD1) + .type("string"), + new Fields.Builder().name(FIELD2) + .type("string"))), + new Summaryclass.Builder() + .id(1) + .name(Execution.ATTRIBUTEPREFETCH) + .fields(Arrays.asList(new Fields.Builder().name(FIELD2) + .type("string"))), + new Summaryclass.Builder() + .id(2) + .name(THIRD_OPTION) + .fields(Arrays.asList(new Fields.Builder().name(FIELD3) + .type("string")))); + + } + + @After + public void tearDown() throws Exception { + searchChain = null; + context = null; + execution = null; + } + + @Test + public final void testTrivial() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD1); + Result result = execution.search(query); + execution.fill(result); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); + assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + } + + @Test + public final void testWithOnlyAttribute() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD2); + Result result = execution.search(query); + execution.fill(result, THIRD_OPTION); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); + assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); + assertTrue(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + } + + @Test + public final void testWithOnlyDiskfieldCorrectClassRequested() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD3); + Result result = execution.search(query); + execution.fill(result, THIRD_OPTION); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); + assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); + assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + } + @Test + public final void testTrivialCaseWithOnlyDiskfieldWrongClassRequested() { + final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD1); + Result result = execution.search(query); + execution.fill(result, THIRD_OPTION); + assertEquals(1, result.getConcreteHitCount()); + assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); + assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); + assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java new file mode 100644 index 00000000000..c9d73853cca --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -0,0 +1,928 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import com.yahoo.component.Version; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.IndexedItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.IndexInfoConfig.Indexinfo; +import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Alias; +import com.yahoo.search.config.IndexInfoConfig.Indexinfo.Command; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.Sorting.AttributeSorter; +import com.yahoo.search.query.Sorting.FieldOrder; +import com.yahoo.search.query.Sorting.LowerCaseSorter; +import com.yahoo.search.query.Sorting.Order; +import com.yahoo.search.query.Sorting.UcaSorter; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Specification for the conversion of YQL+ expressions to Vespa search queries. + * + * @author steinar + * @author stiankri + */ +public class YqlParserTestCase { + + private final YqlParser parser = new YqlParser(new ParserEnvironment()); + + @Test + public void requireThatDefaultsAreSane() { + assertTrue(parser.isQueryParser()); + assertNull(parser.getDocTypes()); + } + + @Test + public void requireThatGroupingStepCanBeParsed() { + assertParse("select foo from bar where baz contains 'cox';", + "baz:cox"); + assertEquals("[]", + toString(parser.getGroupingSteps())); + + assertParse("select foo from bar where baz contains 'cox' " + + "| all(group(a) each(output(count())));", + "baz:cox"); + assertEquals("[[]all(group(a) each(output(count())))]", + toString(parser.getGroupingSteps())); + + assertParse("select foo from bar where baz contains 'cox' " + + "| all(group(a) each(output(count()))) " + + "| all(group(b) each(output(count())));", + "baz:cox"); + assertEquals("[[]all(group(a) each(output(count())))," + + " []all(group(b) each(output(count())))]", + toString(parser.getGroupingSteps())); + } + + @Test + public void requireThatGroupingContinuationCanBeParsed() { + assertParse("select foo from bar where baz contains 'cox' " + + "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count())));", + "baz:cox"); + assertEquals("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))]", + toString(parser.getGroupingSteps())); + + assertParse("select foo from bar where baz contains 'cox' " + + "| [{ 'continuations': ['BCBCBCBEBG', 'BCBKCBACBKCCK'] }]all(group(a) each(output(count()))) " + + "| [{ 'continuations': ['BCBBBBBDBF', 'BCBJBPCBJCCJ'] }]all(group(b) each(output(count())));", + "baz:cox"); + assertEquals("[[BCBCBCBEBG, BCBKCBACBKCCK]all(group(a) each(output(count())))," + + " [BCBBBBBDBF, BCBJBPCBJCCJ]all(group(b) each(output(count())))]", + toString(parser.getGroupingSteps())); + } + + @Test + public void test() { + assertParse("select foo from bar where title contains \"madonna\";", + "title:madonna"); + } + + @Test + public void testOr() { + assertParse("select foo from bar where title contains \"madonna\" or title contains \"saint\";", + "OR title:madonna title:saint"); + assertParse("select foo from bar where title contains \"madonna\" or title contains \"saint\" or title " + + "contains \"angel\";", + "OR title:madonna title:saint title:angel"); + } + + @Test + public void testAnd() { + assertParse("select foo from bar where title contains \"madonna\" and title contains \"saint\";", + "AND title:madonna title:saint"); + assertParse("select foo from bar where title contains \"madonna\" and title contains \"saint\" and title " + + "contains \"angel\";", + "AND title:madonna title:saint title:angel"); + } + + @Test + public void testAndNot() { + assertParse("select foo from bar where title contains \"madonna\" and !(title contains \"saint\");", + "+title:madonna -title:saint"); + } + + @Test + public void testLessThan() { + assertParse("select foo from bar where price < 500;", "price:<500"); + assertParse("select foo from bar where 500 < price;", "price:>500"); + } + + @Test + public void testGreaterThan() { + assertParse("select foo from bar where price > 500;", "price:>500"); + assertParse("select foo from bar where 500 > price;", "price:<500"); + } + + @Test + public void testLessThanOrEqual() { + assertParse("select foo from bar where price <= 500;", "price:[;500]"); + assertParse("select foo from bar where 500 <= price;", "price:[500;]"); + } + + @Test + public void testGreaterThanOrEqual() { + assertParse("select foo from bar where price >= 500;", "price:[500;]"); + assertParse("select foo from bar where 500 >= price;", "price:[;500]"); + } + + @Test + public void testEquality() { + assertParse("select foo from bar where price = 500;", "price:500"); + assertParse("select foo from bar where 500 = price;", "price:500"); + } + + @Test + public void testNegativeLessThan() { + assertParse("select foo from bar where price < -500;", "price:<-500"); + assertParse("select foo from bar where -500 < price;", "price:>-500"); + } + + @Test + public void testNegativeGreaterThan() { + assertParse("select foo from bar where price > -500;", "price:>-500"); + assertParse("select foo from bar where -500 > price;", "price:<-500"); + } + + @Test + public void testNegativeLessThanOrEqual() { + assertParse("select foo from bar where price <= -500;", "price:[;-500]"); + assertParse("select foo from bar where -500 <= price;", "price:[-500;]"); + } + + @Test + public void testNegativeGreaterThanOrEqual() { + assertParse("select foo from bar where price >= -500;", "price:[-500;]"); + assertParse("select foo from bar where -500 >= price;", "price:[;-500]"); + } + + @Test + public void testNegativeEquality() { + assertParse("select foo from bar where price = -500;", "price:-500"); + assertParse("select foo from bar where -500 = price;", "price:-500"); + } + + @Test + public void testAnnotatedLessThan() { + assertParse("select foo from bar where price < ([{\"filter\": true}](-500));", "|price:<-500"); + assertParse("select foo from bar where ([{\"filter\": true}]500) < price;", "|price:>500"); + } + + @Test + public void testAnnotatedGreaterThan() { + assertParse("select foo from bar where price > ([{\"filter\": true}]500);", "|price:>500"); + assertParse("select foo from bar where ([{\"filter\": true}](-500)) > price;", "|price:<-500"); + } + + @Test + public void testAnnotatedLessThanOrEqual() { + assertParse("select foo from bar where price <= ([{\"filter\": true}](-500));", "|price:[;-500]"); + assertParse("select foo from bar where ([{\"filter\": true}]500) <= price;", "|price:[500;]"); + } + + @Test + public void testAnnotatedGreaterThanOrEqual() { + assertParse("select foo from bar where price >= ([{\"filter\": true}]500);", "|price:[500;]"); + assertParse("select foo from bar where ([{\"filter\": true}](-500)) >= price;", "|price:[;-500]"); + } + + @Test + public void testAnnotatedEquality() { + assertParse("select foo from bar where price = ([{\"filter\": true}](-500));", "|price:-500"); + assertParse("select foo from bar where ([{\"filter\": true}]500) = price;", "|price:500"); + } + + @Test + public void testTermAnnotations() { + assertEquals("merkelapp", + getRootWord("select foo from bar where baz contains " + + "([ {\"label\": \"merkelapp\"} ]\"colors\");").getLabel()); + assertEquals("another", + getRootWord("select foo from bar where baz contains " + + "([ {\"annotations\": {\"cox\": \"another\"}} ]\"colors\");").getAnnotation("cox")); + assertEquals(23.0, getRootWord("select foo from bar where baz contains " + + "([ {\"significance\": 23.0} ]\"colors\");").getSignificance(), 1E-6); + assertEquals(23, getRootWord("select foo from bar where baz contains " + + "([ {\"id\": 23} ]\"colors\");").getUniqueID()); + assertEquals(150, getRootWord("select foo from bar where baz contains " + + "([ {\"weight\": 150} ]\"colors\");").getWeight()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"usePositionData\": false} ]\"colors\");").usePositionData()); + assertTrue(getRootWord("select foo from bar where baz contains " + + "([ {\"filter\": true} ]\"colors\");").isFilter()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"ranked\": false} ]\"colors\");").isRanked()); + + Substring origin = getRootWord("select foo from bar where baz contains " + + "([ {\"origin\": {\"original\": \"abc\", \"offset\": 1, \"length\": 2}} ]" + + "\"colors\");").getOrigin(); + assertEquals("abc", origin.string); + assertEquals(1, origin.start); + assertEquals(3, origin.end); + } + + @Test + public void testPhrase() { + assertParse("select foo from bar where baz contains phrase(\"a\", \"b\");", + "baz:\"a b\""); + } + + @Test + public void testNestedPhrase() { + assertParse("select foo from bar where baz contains phrase(\"a\", \"b\", phrase(\"c\", \"d\"));", + "baz:\"a b c d\""); + } + + @Test + public void testNestedPhraseSegment() { + assertParse("select foo from bar where baz contains " + + "phrase(\"a\", \"b\", [ {\"origin\": {\"original\": \"c d\", \"offset\": 0, \"length\": 3}} ]" + + "phrase(\"c\", \"d\"));", + "baz:\"a b 'c d'\""); + } + + @Test + public void testStemming() { + assertTrue(getRootWord("select foo from bar where baz contains " + + "([ {\"stem\": false} ]\"colors\");").isStemmed()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"stem\": true} ]\"colors\");").isStemmed()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "\"colors\";").isStemmed()); + } + + @Test + public void testAccentDropping() { + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"accentDrop\": false} ]\"colors\");").isNormalizable()); + assertTrue(getRootWord("select foo from bar where baz contains " + + "([ {\"accentDrop\": true} ]\"colors\");").isNormalizable()); + assertTrue(getRootWord("select foo from bar where baz contains " + + "\"colors\";").isNormalizable()); + } + + @Test + public void testCaseNormalization() { + assertTrue(getRootWord("select foo from bar where baz contains " + + "([ {\"normalizeCase\": false} ]\"colors\");").isLowercased()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "([ {\"normalizeCase\": true} ]\"colors\");").isLowercased()); + assertFalse(getRootWord("select foo from bar where baz contains " + + "\"colors\";").isLowercased()); + } + + @Test + public void testSegmentingRule() { + assertEquals(SegmentingRule.PHRASE, + getRootWord("select foo from bar where baz contains " + + "([ {\"andSegmenting\": false} ]\"colors\");").getSegmentingRule()); + assertEquals(SegmentingRule.BOOLEAN_AND, + getRootWord("select foo from bar where baz contains " + + "([ {\"andSegmenting\": true} ]\"colors\");").getSegmentingRule()); + assertEquals(SegmentingRule.LANGUAGE_DEFAULT, + getRootWord("select foo from bar where baz contains " + + "\"colors\";").getSegmentingRule()); + } + + @Test + public void testNfkc() { + assertEquals("a\u030a", + getRootWord("select foo from bar where baz contains " + + "([ {\"nfkc\": false} ]\"a\\u030a\");").getWord()); + assertEquals("\u00e5", + getRootWord("select foo from bar where baz contains " + + "([ {\"nfkc\": true} ]\"a\\u030a\");").getWord()); + assertEquals("\u00e5", + getRootWord("select foo from bar where baz contains " + + "\"a\\u030a\";").getWord()); + } + + @Test + public void testImplicitTransforms() { + assertFalse(getRootWord("select foo from bar where baz contains ([ {\"implicitTransforms\": " + + "false} ]\"cox\");").isFromQuery()); + assertTrue(getRootWord("select foo from bar where baz contains ([ {\"implicitTransforms\": " + + "true} ]\"cox\");").isFromQuery()); + assertTrue(getRootWord("select foo from bar where baz contains \"cox\";").isFromQuery()); + } + + @Test + public void testConnectivity() { + QueryTree parsed = parse("select foo from bar where " + + "title contains ([{\"id\": 1, \"connectivity\": {\"id\": 3, \"weight\": 7.0}}]\"madonna\") " + + "and title contains ([{\"id\": 2}]\"saint\") " + + "and title contains ([{\"id\": 3}]\"angel\");"); + assertEquals("AND title:madonna title:saint title:angel", + parsed.toString()); + AndItem root = (AndItem)parsed.getRoot(); + WordItem first = (WordItem)root.getItem(0); + WordItem second = (WordItem)root.getItem(1); + WordItem third = (WordItem)root.getItem(2); + assertTrue(first.getConnectedItem() == third); + assertEquals(first.getConnectivity(), 7.0d, 1E-6); + assertNull(second.getConnectedItem()); + + assertParseFail("select foo from bar where " + + "title contains ([{\"id\": 1, \"connectivity\": {\"id\": 4, \"weight\": 7.0}}]\"madonna\") " + + "and title contains ([{\"id\": 2}]\"saint\") " + + "and title contains ([{\"id\": 3}]\"angel\");", + new NullPointerException("Item 'title:madonna' was specified to connect to item with ID 4, " + + "which does not exist in the query.")); + } + + @Test + public void testAnnotatedPhrase() { + QueryTree parsed = + parse("select foo from bar where baz contains ([{\"label\": \"hello world\"}]phrase(\"a\", \"b\"));"); + assertEquals("baz:\"a b\"", parsed.toString()); + PhraseItem phrase = (PhraseItem)parsed.getRoot(); + assertEquals("hello world", phrase.getLabel()); + } + + @Test + public void testRange() { + QueryTree parsed = parse("select foo from bar where range(baz,1,8);"); + assertEquals("baz:[1;8]", parsed.toString()); + } + + @Test + public void testNegativeRange() { + QueryTree parsed = parse("select foo from bar where range(baz,-8,-1);"); + assertEquals("baz:[-8;-1]", parsed.toString()); + } + + @Test + public void testRangeIllegalArguments() { + assertParseFail("select foo from bar where range(baz,cox,8);", + new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.")); + } + + @Test + public void testNear() { + assertParse("select foo from bar where description contains near(\"a\", \"b\");", + "NEAR(2) description:a description:b"); + assertParse("select foo from bar where description contains ([ {\"distance\": 100} ]near(\"a\", \"b\"));", + "NEAR(100) description:a description:b"); + } + + @Test + public void testOrderedNear() { + assertParse("select foo from bar where description contains onear(\"a\", \"b\");", + "ONEAR(2) description:a description:b"); + assertParse("select foo from bar where description contains ([ {\"distance\": 100} ]onear(\"a\", \"b\"));", + "ONEAR(100) description:a description:b"); + } + + //This test is order dependent. Fix this!! + @Test + public void testWand() { + assertParse("select foo from bar where wand(description, {\"a\":1, \"b\":2});", + "WAND(10,0.0,1.0) description{[1]:\"a\",[2]:\"b\"}"); + assertParse("select foo from bar where [ {\"scoreThreshold\": 13.3, \"targetNumHits\": 7, " + + "\"thresholdBoostFactor\": 2.3} ]wand(description, {\"a\":1, \"b\":2});", + "WAND(7,13.3,2.3) description{[1]:\"a\",[2]:\"b\"}"); + } + + @Test + public void testNumericWand() { + String numWand = "WAND(10,0.0,1.0) description{[1]:\"11\",[2]:\"37\"}"; + assertParse("select foo from bar where wand(description, [[11,1], [37,2]]);", numWand); + assertParse("select foo from bar where wand(description, [[11L,1], [37L,2]]);", numWand); + assertParseFail("select foo from bar where wand(description, 12);", + new IllegalArgumentException("Expected ARRAY or MAP, got LITERAL.")); + } + + @Test + //This test is order dependent. Fix it! + public void testWeightedSet() { + assertParse("select foo from bar where weightedSet(description, {\"a\":1, \"b\":2});", + "WEIGHTEDSET description{[1]:\"a\",[2]:\"b\"}"); + assertParseFail("select foo from bar where weightedSet(description, {\"a\":g, \"b\":2});", + new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD.")); + assertParseFail("select foo from bar where weightedSet(description);", + new IllegalArgumentException("Expected 2 arguments, got 1.")); + } + + //This test is order dependent. Fix it! + @Test + public void testDotProduct() { + assertParse("select foo from bar where dotProduct(description, {\"a\":1, \"b\":2});", + "DOTPRODUCT description{[1]:\"a\",[2]:\"b\"}"); + assertParse("select foo from bar where dotProduct(description, {\"a\":2});", + "DOTPRODUCT description{[2]:\"a\"}"); + } + + @Test + public void testPredicate() { + assertParse("select foo from bar where predicate(predicate_field, " + + "{\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23L});", + "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23"); + assertParse("select foo from bar where predicate(predicate_field, " + + "{\"gender\":\"male\", \"hobby\":[\"music\", \"hiking\"]}, {\"age\":23});", + "PREDICATE_QUERY_ITEM gender=male, hobby=music, hobby=hiking, age:23"); + assertParse("select foo from bar where predicate(predicate_field, 0, void);", + "PREDICATE_QUERY_ITEM "); + } + + @Test + public void testPredicateWithSubQueries() { + assertParse("select foo from bar where predicate(predicate_field, " + + "{\"0x03\":{\"gender\":\"male\"},\"0x01\":{\"hobby\":[\"music\", \"hiking\"]}}, {\"0x80ffffffffffffff\":{\"age\":23L}});", + "PREDICATE_QUERY_ITEM gender=male[0x3], hobby=music[0x1], hobby=hiking[0x1], age:23[0x80ffffffffffffff]"); + assertParseFail("select foo from bar where predicate(foo, null, {\"0x80000000000000000\":{\"age\":23}});", + new NumberFormatException("Too long subquery string: 0x80000000000000000")); + assertParse("select foo from bar where predicate(predicate_field, " + + "{\"[0,1]\":{\"gender\":\"male\"},\"[0]\":{\"hobby\":[\"music\", \"hiking\"]}}, {\"[62, 63]\":{\"age\":23L}});", + "PREDICATE_QUERY_ITEM gender=male[0x3], hobby=music[0x1], hobby=hiking[0x1], age:23[0xc000000000000000]"); + } + + @Test + public void testRank() { + assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\");", + "RANK a:A b:B"); + assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\", c " + + "contains \"C\");", + "RANK a:A b:B c:C"); + assertParse("select foo from bar where rank(a contains \"A\", b contains \"B\" or c " + + "contains \"C\");", + "RANK a:A (OR b:B c:C)"); + } + + @Test + public void testWeakAnd() { + assertParse("select foo from bar where weakAnd(a contains \"A\", b contains \"B\");", + "WAND(100) a:A b:B"); + assertParse("select foo from bar where [{\"targetNumHits\": 37}]weakAnd(a contains \"A\", " + + "b contains \"B\");", + "WAND(37) a:A b:B"); + + QueryTree tree = parse("select foo from bar where [{\"scoreThreshold\": 41}]weakAnd(a " + + "contains \"A\", b contains \"B\");"); + assertEquals("WAND(100) a:A b:B", tree.toString()); + assertEquals(WeakAndItem.class, tree.getRoot().getClass()); + assertEquals(41, ((WeakAndItem)tree.getRoot()).getScoreThreshold()); + } + + @Test + public void testEquiv() { + assertParse("select foo from bar where fieldName contains equiv(\"A\",\"B\");", + "EQUIV fieldName:A fieldName:B"); + assertParse("select foo from bar where fieldName contains " + + "equiv(\"ny\",phrase(\"new\",\"york\"));", + "EQUIV fieldName:ny fieldName:\"new york\""); + assertParseFail("select foo from bar where fieldName contains equiv(\"ny\");", + new IllegalArgumentException("Expected 2 or more arguments, got 1.")); + assertParseFail("select foo from bar where fieldName contains equiv(\"ny\", nalle(void));", + new IllegalArgumentException("Expected function 'phrase', got 'nalle'.")); + assertParseFail("select foo from bar where fieldName contains equiv(\"ny\", 42);", + new ClassCastException("Cannot cast java.lang.Integer to java.lang.String")); + } + + @Test + public void testAffixItems() { + assertRootClass("select foo from bar where baz contains ([ {\"suffix\": true} ]\"colors\");", + SuffixItem.class); + assertRootClass("select foo from bar where baz contains ([ {\"prefix\": true} ]\"colors\");", + PrefixItem.class); + assertRootClass("select foo from bar where baz contains ([ {\"substring\": true} ]\"colors\");", + SubstringItem.class); + assertParseFail("select foo from bar where description contains ([ {\"suffix\": true, " + + "\"prefix\": true} ]\"colors\");", + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + assertParseFail("select foo from bar where description contains ([ {\"suffix\": true, " + + "\"substring\": true} ]\"colors\");", + new IllegalArgumentException("Only one of prefix, substring and suffix can be set.")); + } + + @Test + public void testLongNumberInSimpleExpression() { + assertParse("select foo from bar where price = 8589934592L;", + "price:8589934592"); + } + + @Test + public void testNegativeLongNumberInSimpleExpression() { + assertParse("select foo from bar where price = -8589934592L;", + "price:-8589934592"); + } + + @Test + public void testSources() { + assertSources("select foo from sourceA where price <= 500;", + Arrays.asList("sourceA")); + } + + @Test + public void testWildCardSources() { + assertSources("select foo from sources * where price <= 500;", + Collections.<String>emptyList()); + } + + @Test + public void testMultiSources() { + assertSources("select foo from sources sourceA, sourceB where price <= 500;", + Arrays.asList("sourceA", "sourceB")); + } + + @Test + public void testFields() { + assertSummaryFields("select fieldA from bar where price <= 500;", + Arrays.asList("fieldA")); + assertSummaryFields("select fieldA, fieldB from bar where price <= 500;", + Arrays.asList("fieldA", "fieldB")); + assertSummaryFields("select fieldA, fieldB, fieldC from bar where price <= 500;", + Arrays.asList("fieldA", "fieldB", "fieldC")); + assertSummaryFields("select * from bar where price <= 500;", + Collections.<String>emptyList()); + } + + @Test + public void testFieldsRoot() { + assertParse("select * from bar where price <= 500;", + "price:[;500]"); + } + + @Test + public void testOffset() { + assertParse("select foo from bar where title contains \"madonna\" offset 37;", + "title:madonna"); + assertEquals(Integer.valueOf(37), parser.getOffset()); + } + + @Test + public void testLimit() { + assertParse("select foo from bar where title contains \"madonna\" limit 29;", + "title:madonna"); + assertEquals(Integer.valueOf(29), parser.getHits()); + } + + @Test + public void testOffsetAndLimit() { + assertParse("select foo from bar where title contains \"madonna\" limit 31 offset 29;", + "title:madonna"); + assertEquals(Integer.valueOf(29), parser.getOffset()); + assertEquals(Integer.valueOf(2), parser.getHits()); + + assertParse("select * from bar where title contains \"madonna\" limit 41 offset 37;", + "title:madonna"); + assertEquals(Integer.valueOf(37), parser.getOffset()); + assertEquals(Integer.valueOf(4), parser.getHits()); + } + + @Test + public void testTimeout() { + assertParse("select * from bar where title contains \"madonna\" timeout 7;", + "title:madonna"); + assertEquals(Integer.valueOf(7), parser.getTimeout()); + + assertParse("select foo from bar where title contains \"madonna\" limit 600 timeout 3;", + "title:madonna"); + assertEquals(Integer.valueOf(3), parser.getTimeout()); + } + + @Test + public void testOrdering() { + assertParse("select foo from bar where title contains \"madonna\" order by something asc, " + + "shoesize desc limit 600 timeout 3;", + "title:madonna"); + assertEquals(2, parser.getSorting().fieldOrders().size()); + assertEquals("something", parser.getSorting().fieldOrders().get(0).getFieldName()); + assertEquals(Order.ASCENDING, parser.getSorting().fieldOrders().get(0).getSortOrder()); + assertEquals("shoesize", parser.getSorting().fieldOrders().get(1).getFieldName()); + assertEquals(Order.DESCENDING, parser.getSorting().fieldOrders().get(1).getSortOrder()); + + assertParse("select foo from bar where title contains \"madonna\" order by other limit 600 " + + "timeout 3;", + "title:madonna"); + assertEquals("other", parser.getSorting().fieldOrders().get(0).getFieldName()); + assertEquals(Order.ASCENDING, parser.getSorting().fieldOrders().get(0).getSortOrder()); + } + + @Test + public void testAnnotatedOrdering() { + assertParse( + "select foo from bar where title contains \"madonna\"" + + " order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc" + + " limit 600" + " timeout 3;", "title:madonna"); + final FieldOrder fieldOrder = parser.getSorting().fieldOrders().get(0); + assertEquals("other", fieldOrder.getFieldName()); + assertEquals(Order.DESCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(UcaSorter.class, sorter.getClass()); + final UcaSorter uca = (UcaSorter) sorter; + assertEquals("en_US", uca.getLocale()); + assertEquals(UcaSorter.Strength.IDENTICAL, uca.getStrength()); + } + + @Test + public void testMultipleAnnotatedOrdering() { + assertParse( + "select foo from bar where title contains \"madonna\"" + + " order by [{\"function\": \"uca\", \"locale\": \"en_US\", \"strength\": \"IDENTICAL\"}]other desc," + + " [{\"function\": \"lowercase\"}]something asc" + + " limit 600" + " timeout 3;", "title:madonna"); + { + final FieldOrder fieldOrder = parser.getSorting().fieldOrders() + .get(0); + assertEquals("other", fieldOrder.getFieldName()); + assertEquals(Order.DESCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(UcaSorter.class, sorter.getClass()); + final UcaSorter uca = (UcaSorter) sorter; + assertEquals("en_US", uca.getLocale()); + assertEquals(UcaSorter.Strength.IDENTICAL, uca.getStrength()); + } + { + final FieldOrder fieldOrder = parser.getSorting().fieldOrders() + .get(1); + assertEquals("something", fieldOrder.getFieldName()); + assertEquals(Order.ASCENDING, fieldOrder.getSortOrder()); + final AttributeSorter sorter = fieldOrder.getSorter(); + assertEquals(LowerCaseSorter.class, sorter.getClass()); + } + } + + @Test + public void testSegmenting() { + assertParse("select * from bar where ([{\"segmenter\": {\"version\": \"58.67.49\", \"backend\": " + + "\"yell\"}}] title contains \"madonna\");", + "title:madonna"); + assertEquals("yell", parser.getSegmenterBackend()); + assertEquals(new Version("58.67.49"), parser.getSegmenterVersion()); + + assertParse("select * from bar where ([{\"segmenter\": {\"version\": \"8.7.3\", \"backend\": " + + "\"yell\"}}]([{\"targetNumHits\": 9999438}] weakAnd(format contains \"online\", title contains " + + "\"madonna\")));", + "WAND(9999438) format:online title:madonna"); + assertEquals("yell", parser.getSegmenterBackend()); + assertEquals(new Version("8.7.3"), parser.getSegmenterVersion()); + + assertParse("select * from bar where [{\"segmenter\": {\"version\": \"18.47.39\", \"backend\": " + + "\"yell\"}}] ([{\"targetNumHits\": 99909438}] weakAnd(format contains \"online\", title contains " + + "\"madonna\"));", + "WAND(99909438) format:online title:madonna"); + assertEquals("yell", parser.getSegmenterBackend()); + assertEquals(new Version("18.47.39"), parser.getSegmenterVersion()); + + assertParse("select * from bar where [{\"targetNumHits\": 99909438}] weakAnd(format contains " + + "\"online\", title contains \"madonna\");", + "WAND(99909438) format:online title:madonna"); + assertNull(parser.getSegmenterBackend()); + assertNull(parser.getSegmenterVersion()); + + assertParse("select * from bar where [{\"segmenter\": {\"version\": \"58.67.49\", \"backend\": " + + "\"yell\"}}](title contains \"madonna\") order by shoesize;", + "title:madonna"); + assertEquals("yell", parser.getSegmenterBackend()); + assertEquals(new Version("58.67.49"), parser.getSegmenterVersion()); + } + + @Test + public void testNegativeHitLimit() { + assertParse( + "select * from sources * where [{\"hitLimit\": -38}]range(foo, 0, 1);", + "foo:[0;1;-38]"); + } + + @Test + public void testRangeSearchHitPopulationOrdering() { + assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true}]range(foo, 0, 1);", "foo:[0;1;38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": false}]range(foo, 0, 1);", "foo:[0;1;-38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": true}]range(foo, 0, 1);", "foo:[0;1;-38]"); + assertParse("select * from sources * where [{\"hitLimit\": 38, \"descending\": false}]range(foo, 0, 1);", "foo:[0;1;38]"); + + boolean gotExceptionFromParse = false; + try { + parse("select * from sources * where [{\"hitLimit\": 38, \"ascending\": true, \"descending\": false}]range(foo, 0, 1);"); + } catch (IllegalArgumentException e) { + assertTrue("Expected information about abuse of settings.", + e.getMessage().contains("both ascending and descending ordering set")); + gotExceptionFromParse = true; + } + assertTrue(gotExceptionFromParse); + } + + @Test + public void testOpenIntervals() { + assertParse("select * from sources * where range(title, 0.0, 500.0);", + "title:[0.0;500.0]"); + assertParse( + "select * from sources * where [{\"bounds\": \"open\"}]range(title, 0.0, 500.0);", + "title:<0.0;500.0>"); + assertParse( + "select * from sources * where [{\"bounds\": \"leftOpen\"}]range(title, 0.0, 500.0);", + "title:<0.0;500.0]"); + assertParse( + "select * from sources * where [{\"bounds\": \"rightOpen\"}]range(title, 0.0, 500.0);", + "title:[0.0;500.0>"); + } + + @Test + public void testInheritedAnnotations() { + { + QueryTree x = parse("select * from sources * where ([{\"ranked\": false}](foo contains \"a\" and bar contains \"b\")) or foor contains ([{\"ranked\": false}]\"c\");"); + List<IndexedItem> terms = QueryTree.getPositiveTerms(x); + assertEquals(3, terms.size()); + for (IndexedItem term : terms) { + assertFalse(((Item) term).isRanked()); + } + } + { + QueryTree x = parse("select * from sources * where [{\"ranked\": false}](foo contains \"a\" and bar contains \"b\");"); + List<IndexedItem> terms = QueryTree.getPositiveTerms(x); + assertEquals(2, terms.size()); + for (IndexedItem term : terms) { + assertFalse(((Item) term).isRanked()); + } + } + } + + @Test + public void testMoreInheritedAnnotations() { + final String yqlQuery = "select * from sources * where " + + "([{\"ranked\": false}](foo contains \"a\" " + + "and ([{\"ranked\": true}](bar contains \"b\" " + + "or ([{\"ranked\": false}](foo contains \"c\" " + + "and foo contains ([{\"ranked\": true}]\"d\")))))));"; + QueryTree x = parse(yqlQuery); + List<IndexedItem> terms = QueryTree.getPositiveTerms(x); + assertEquals(4, terms.size()); + for (IndexedItem term : terms) { + switch (term.getIndexedString()) { + case "a": + case "c": + assertFalse(((Item) term).isRanked()); + break; + case "b": + case "d": + assertTrue(((Item) term).isRanked()); + break; + default: + fail(); + } + } + } + + @Test + public void testFieldAliases() { + IndexInfoConfig modelConfig = new IndexInfoConfig(new IndexInfoConfig.Builder().indexinfo(new Indexinfo.Builder() + .name("music").command(new Command.Builder().indexname("title").command("index")) + .alias(new Alias.Builder().alias("song").indexname("title")))); + IndexModel model = new IndexModel(modelConfig, (QrSearchersConfig)null); + + IndexFacts indexFacts = new IndexFacts(model); + ParserEnvironment parserEnvironment = new ParserEnvironment().setIndexFacts(indexFacts); + YqlParser configuredParser = new YqlParser(parserEnvironment); + QueryTree x = configuredParser.parse(new Parsable() + .setQuery("select * from sources * where title contains \"a\" and song contains \"b\";")); + List<IndexedItem> terms = QueryTree.getPositiveTerms(x); + assertEquals(2, terms.size()); + for (IndexedItem term : terms) { + assertEquals("title", term.getIndexName()); + } + } + + @Test + public void testRegexp() { + QueryTree x = parse("select * from sources * where foo matches \"a b\";"); + Item root = x.getRoot(); + assertSame(RegExpItem.class, root.getClass()); + assertEquals("a b", ((RegExpItem) root).stringValue()); + } + + @Test + public void testWordAlternatives() { + QueryTree x = parse("select * from sources * where foo contains alternatives({\"trees\": 1.0, \"tree\": 0.7});"); + Item root = x.getRoot(); + assertSame(WordAlternativesItem.class, root.getClass()); + WordAlternativesItem alternatives = (WordAlternativesItem) root; + checkWordAlternativesContent(alternatives); + } + + @Test + public void testWordAlternativesWithOrigin() { + QueryTree x = parse("select * from sources * where foo contains" + + " ([{\"origin\": {\"original\": \" trees \", \"offset\": 1, \"length\": 5}}]" + + "alternatives({\"trees\": 1.0, \"tree\": 0.7}));"); + Item root = x.getRoot(); + assertSame(WordAlternativesItem.class, root.getClass()); + WordAlternativesItem alternatives = (WordAlternativesItem) root; + checkWordAlternativesContent(alternatives); + Substring origin = alternatives.getOrigin(); + assertEquals(1, origin.start); + assertEquals(6, origin.end); + assertEquals("trees", origin.getValue()); + assertEquals(" trees ", origin.getSuperstring()); + } + + @Test + public void testWordAlternativesInPhrase() { + QueryTree x = parse("select * from sources * where" + + " foo contains phrase(\"forest\", alternatives({\"trees\": 1.0, \"tree\": 0.7}));"); + Item root = x.getRoot(); + assertSame(PhraseItem.class, root.getClass()); + PhraseItem phrase = (PhraseItem) root; + assertEquals(2, phrase.getItemCount()); + assertEquals("forest", ((WordItem) phrase.getItem(0)).getWord()); + checkWordAlternativesContent((WordAlternativesItem) phrase.getItem(1)); + } + + private void checkWordAlternativesContent(WordAlternativesItem alternatives) { + boolean seenTree = false; + boolean seenForest = false; + final String forest = "trees"; + final String tree = "tree"; + assertEquals(2, alternatives.getAlternatives().size()); + for (WordAlternativesItem.Alternative alternative : alternatives.getAlternatives()) { + if (tree.equals(alternative.word)) { + assertFalse("Duplicate term introduced", seenTree); + seenTree = true; + assertEquals(.7d, alternative.exactness, 1e-15d); + } else if (forest.equals(alternative.word)) { + assertFalse("Duplicate term introduced", seenForest); + seenForest = true; + assertEquals(1.0d, alternative.exactness, 1e-15d); + } else { + fail("Unexpected term: " + alternative.word); + } + } + } + + private void assertParse(String yqlQuery, String expectedQueryTree) { + assertEquals(expectedQueryTree, parse(yqlQuery).toString()); + } + + private void assertParseFail(String yqlQuery, Throwable expectedException) { + try { + parse(yqlQuery); + } catch (Throwable t) { + assertEquals(expectedException.getClass(), t.getClass()); + assertEquals(expectedException.getMessage(), t.getMessage()); + return; + } + fail("Parse succeeded: " + yqlQuery); + } + + private void assertSources(String yqlQuery, Collection<String> expectedSources) { + parse(yqlQuery); + assertEquals(new HashSet<>(expectedSources), parser.getYqlSources()); + } + + private void assertSummaryFields(String yqlQuery, Collection<String> expectedSummaryFields) { + parse(yqlQuery); + assertEquals(new HashSet<>(expectedSummaryFields), parser.getYqlSummaryFields()); + } + + private WordItem getRootWord(String yqlQuery) { + Item root = parse(yqlQuery).getRoot(); + assertTrue(root instanceof WordItem); + return (WordItem)root; + } + + private void assertRootClass(String yqlQuery, Class<? extends Item> expectedRootClass) { + assertEquals(expectedRootClass, parse(yqlQuery).getRoot().getClass()); + } + + private QueryTree parse(String yqlQuery) { + return parser.parse(new Parsable().setQuery(yqlQuery)); + } + + private static String toString(List<VespaGroupingStep> steps) { + List<String> actual = new ArrayList<>(steps.size()); + for (VespaGroupingStep step : steps) { + actual.add(step.continuations().toString() + + step.getOperation()); + } + return actual.toString(); + } +} |