diff options
3 files changed, 0 insertions, 351 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java index e09d47e7bc1..24cb1a16d08 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java @@ -73,7 +73,6 @@ public class BundleMapper { bundleFromClass.put("com.yahoo.prelude.searcher.FillSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.JSONDebugSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.JuniperSearcher", searchAndDocprocBundle); - bundleFromClass.put("com.yahoo.prelude.searcher.KeyValueSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.MultipleResultsSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.PosSearcher", searchAndDocprocBundle); bundleFromClass.put("com.yahoo.prelude.searcher.QuerySnapshotSearcher", searchAndDocprocBundle); diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/KeyValueSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/KeyValueSearcher.java deleted file mode 100644 index a282dc22b53..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/KeyValueSearcher.java +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher; - -import com.yahoo.document.BucketId; -import com.yahoo.document.BucketIdFactory; -import com.yahoo.document.DocumentId; -import com.yahoo.document.GlobalId; -import com.yahoo.document.idstring.IdString; -import com.yahoo.documentapi.messagebus.protocol.SearchColumnPolicy; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.query.IntItem; -import com.yahoo.prelude.query.QueryCanonicalizer; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.grouping.vespa.GroupingExecutor; -import com.yahoo.search.query.Model; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.result.DefaultErrorHit; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.vdslib.BucketDistribution; -import com.yahoo.component.chain.dependencies.Before; - -import java.util.Iterator; -import java.util.logging.Logger; - - -/** - * Searcher that does efficient key/value lookup using Vespa search as a - * backend. It does so by bypassing the first phase ranking, and only performs - * the second phase summary fetching. - * - * The keys to find are input as a comma-seprated list using the <i>keys</i> - * query parameter. Each key should match a part of a document id. Given the key - * 'foo', and document id namespace 'mynamespace', the document id matched will - * be 'id:mynamespace:keyvalue::foo'. - * - * To scale the throughput with the number of partitions, the searcher uses the - * same hashing mechanisms as the document API to find out which node each key - * belongs to. The searcher then dispatches a summary request to retrieve keys - * and returns the result. - * - * @author <a href="lulf@yahoo-inc.com">Ulf Lilleengen</a> - */ -@Before(GroupingExecutor.COMPONENT_NAME) -public class KeyValueSearcher extends Searcher { - - private static final Logger log = Logger.getLogger(KeyValueSearcher.class.getName()); - private final BucketIdFactory factory = new BucketIdFactory(); - private final BucketDistribution distribution; - private final String summaryClass; - private final String idSchemePrefix; - private final int numRowBits; - private final int traceLevel = 5; - - public KeyValueSearcher(KeyvalueConfig config) { - this.summaryClass = config.summaryName(); - this.idSchemePrefix = createIdSchemePrefix(config); - this.distribution = new BucketDistribution(config.numparts(), SearchColumnPolicy.DEFAULT_NUM_BUCKET_BITS); - this.numRowBits = calcNumRowBits(config.numrows()); - log.config("Configuring " + KeyValueSearcher.class.getName() + " with " + config.numparts() + " partitions and doc id scheme '" + idSchemePrefix + "'"); - } - - private String createIdSchemePrefix(KeyvalueConfig config) { - if (config.docIdScheme().equals(KeyvalueConfig.DocIdScheme.Enum.DOC_SCHEME)) { - return "doc:" + config.docIdNameSpace() + ":"; - } else { - return "id:" + config.docIdNameSpace() + ":" + config.docIdType() + "::"; - } - } - - public Hit createHit(Query query, String key) { - String docId = createDocId(key.trim()); - BucketId id = factory.getBucketId(new DocumentId(docId)); - int partition = getPartition(id); - - FastHit hit = new FastHit(); - hit.setGlobalId(new GlobalId(IdString.createIdString(docId))); - hit.setQuery(query); - hit.setFillable(); - hit.setCached(false); - hit.setPartId(partition << numRowBits, numRowBits); - hit.setRelevance(1.0); - hit.setIgnoreRowBits(true); - hit.setDistributionKey(42); - return hit; - } - - private String createDocId(String key) { - return idSchemePrefix + key; - } - - - @Override - public Result search(Query query, Execution execution) { - String keyProp = query.properties().getString("keys"); - query.getPresentation().setSummary(summaryClass); - if (keyProp == null || keyProp.length() == 0) { - return new Result(query, new ErrorMessage(ErrorMessage.NULL_QUERY, "'keys' parameter not set or empty.")); - } - String[] keyList = keyProp.split(","); - Model model = query.getModel(); - QueryTree tree = model.getQueryTree(); - QueryCanonicalizer.canonicalize(tree); - if (tree.isEmpty()) { - tree.setRoot(new IntItem(String.valueOf(keyProp.hashCode()))); - } - - Result result = new Result(query); - for (String key : keyList) { - result.hits().add(createHit(query, key)); - } - execution.fill(result, summaryClass); - if (query.isTraceable(traceLevel)) { - traceResult(query, result); - } - int totalHits = 0; - Iterator<Hit> hitIterator = result.hits().iterator(); - while (hitIterator.hasNext()) { - Hit hit = hitIterator.next(); - if (hit.isFillable() && hit.isFilled(summaryClass)) { - totalHits++; - } else { - hitIterator.remove(); - } - } - if (totalHits != keyList.length) { - ErrorMessage error = new ErrorMessage(1, "Some keys could not be fetched"); - result.hits().setError(error); - } - result.setTotalHitCount(totalHits); - return result; - } - - private void traceResult(Query query, Result result) { - Iterator<Hit> hitIterator = result.hits().iterator(); - while (hitIterator.hasNext()) { - Hit hit = hitIterator.next(); - if (hit.isFillable() && hit.isFilled(summaryClass)) { - query.trace("Found filled hit: " + hit, traceLevel); - } else { - query.trace("Found hit that was not filled/fillable: " + hit, traceLevel); - } - } - query.trace("Error hit: " + result.hits().getErrorHit(), traceLevel); - } - - private int getPartition(BucketId bucketId) { - return distribution.getColumn(bucketId); - } - - private static int calcNumRowBits(int numRows) { - if (numRows < 1) { - throw new IllegalArgumentException(); - } - for (int i = 0; i < 30; ++i) { - if (numRows - 1 < 1 << i) { - return i; - } - } - return 31; - } -} diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java deleted file mode 100644 index 6a329185ef1..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/KeyValueSearcherTest.java +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.searcher.test; - -import com.yahoo.component.chain.Chain; -import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.document.GlobalId; -import com.yahoo.document.idstring.IdString; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.searcher.KeyValueSearcher; -import com.yahoo.prelude.searcher.KeyvalueConfig; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import org.junit.Before; -import org.junit.Test; - -import java.util.*; -import java.util.Map.Entry; - -import static org.junit.Assert.*; - -public class KeyValueSearcherTest { - - private static class BackendMockup extends Searcher { - private final Map<GlobalId, Entry<String, String>> dataMap; - private final String summaryType; - - public BackendMockup(Map<GlobalId, Entry<String, String>> dataMap, String summaryType) { - this.dataMap = dataMap; - this.summaryType = summaryType; - } - - @Override - public Result search(Query query, Execution execution) { - fail("Should not do search against backend"); - return null; - } - - @Override - public void fill(Result result, String summaryClass, Execution execution) { - if (containsNullItem(result.getQuery().getModel().getQueryTree().getRoot())) - fail("Got a query with a NullItem root. This cannot be encoded."); - int numEmpty = 0; - for (Hit hit : result.hits()) { - FastHit fhit = (FastHit) hit; - Entry<String, String> data = dataMap.get(fhit.getGlobalId()); - if (data != null) { - fhit.setField(data.getKey(), data.getValue()); - fhit.setFilled(summaryType); - } else { - numEmpty++; - } - } - if (numEmpty > 0) { - result.hits().addError(ErrorMessage.createBackendCommunicationError("One or more hits were not filled")); - } - } - } - - private Map<GlobalId, Entry<String,String>> dataMap; - private BackendMockup backend; - @Before - public void setupBackend() { - dataMap = new HashMap<>(); - dataMap.put(new GlobalId(IdString.createIdString("id:keyvalue:keyvalue::foo")), new AbstractMap.SimpleEntry<>("foo", "foovalue")); - dataMap.put(new GlobalId(IdString.createIdString("id:keyvalue:keyvalue::bar")), new AbstractMap.SimpleEntry<>("bar", "barvalue")); - dataMap.put(new GlobalId(IdString.createIdString("id:keyvalue:keyvalue::this_must_be_a_key_in_part1_fsadfasdfa")), new AbstractMap.SimpleEntry<>("this_must_be_a_key_in_part1_fsadfasdfa", "blabla")); - backend = new BackendMockup(dataMap, "mysummary"); - } - - @Test - public void testKeyValueSearcher() { - Result result = executeQuery(getConfigString(1), "?keys=foo,bar"); - assertEquals(2, result.getTotalHitCount()); - for (Hit hit : result.hits()) { - FastHit fhit = (FastHit)hit; - Entry<String, String> data = dataMap.get(fhit.getGlobalId()); - assertEquals(data.getValue(), hit.getField(data.getKey())); - assertTrue(hit.isFilled("mysummary")); - } - - result = executeQuery(getConfigString(1), - "?keys=blabla,fofo", new BackendMockup(dataMap, "mysummary")); - assertEquals(0, result.getTotalHitCount()); - - result = executeQuery(getConfigString(1), - "?keys=non,foo,slsl", new BackendMockup(dataMap, "mysummary")); - assertEquals(1, result.getTotalHitCount()); - } - - @Test - public void testKeyValueSearcherWithNullItemAsQuery() { - Query query = new Query("?keys=foo,bar"); - AndItem and = new AndItem(); - and.addItem(new NullItem()); - query.getModel().getQueryTree().setRoot(and); - Result result = executeQuery(getConfigString(1), query); - assertEquals(2, result.getTotalHitCount()); - } - - private static String getConfigString(int numRows) { - return "raw:numparts 2\nsummaryName \"mysummary\"\ndocIdType \"keyvalue\"\ndocIdNameSpace \"keyvalue\"\nnumrows " + numRows + "\n"; - } - - @Test - public void requireThatIgnoreRowBitsIsEnabledInGeneratedHits() { - Result result = executeQuery(getConfigString(1), - "?keys=foo,bar"); - for (Hit hit : result.hits()) { - FastHit fastHit = (FastHit)hit; - assertTrue(fastHit.shouldIgnoreRowBits()); - } - } - - @Test - public void requireThatNumRowsIsAPositiveNumber() { - for (int i = -10; i < 1; ++i) { - try { - newKeyValueSearcher(getConfigString(i)); - fail(); - } catch (IllegalArgumentException e) { - - } - } - for (int i = 1; i < 10; ++i) { - assertNotNull(newKeyValueSearcher(getConfigString(i))); - } - } - - @Test - public void requireThatNumRowBitsAreCalculatedCorrectly() { - assertRowBits(1, 0); - assertRowBits(2, 1); - assertRowBits(3, 2); - assertRowBits(4, 2); - assertRowBits(5, 3); - assertRowBits(10, 4); - assertRowBits(100, 7); - assertRowBits(1000, 10); - } - - private void assertRowBits(int numRows, int expectedNumRowBits) { - Result result = executeQuery(getConfigString(numRows), "?keys=this_must_be_a_key_in_part1_fsadfasdfa"); - assertEquals(1, result.hits().size()); - FastHit hit = (FastHit)result.hits().get(0); - assertEquals(0, hit.getPartId() & ((1 << expectedNumRowBits) - 1)); - assertEquals(1, hit.getPartId() >> expectedNumRowBits); - } - - private Result executeQuery(String configId, String queryString, Searcher... searchers) { - return executeQuery(configId, new Query(queryString), searchers); - } - - private Result executeQuery(String configId, Query query, Searcher... searchers) { - List<Searcher> chain = new LinkedList<>(); - chain.add(newKeyValueSearcher(configId)); - chain.addAll(Arrays.asList(searchers)); - chain.add(backend); - return new Execution(new Chain<>(chain), Execution.Context.createContextStub()).search(query); - } - - - private static KeyValueSearcher newKeyValueSearcher(String configId) { - return new KeyValueSearcher(new ConfigGetter<>(KeyvalueConfig.class).getConfig(configId)); - } - - private static boolean containsNullItem(Item item) { - if (item instanceof NullItem) return true; - if (item instanceof CompositeItem) { - for (Iterator<Item> i = ((CompositeItem)item).getItemIterator(); i.hasNext(); ) - if (containsNullItem(i.next())) - return true; - } - return false; - } - -} |