diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2018-05-27 21:43:52 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-05-27 21:43:52 +0200 |
commit | 448da3df15fcc4c2bafe93161dfde9a7ac143144 (patch) | |
tree | 351fef364327b3c8e04916cc1475c6c062fa8359 | |
parent | 1d3fd26127c2bad1322cb2b8788fe9273f6d3177 (diff) | |
parent | e23246f9f33103d09f012f4d8cb089ebc6f82698 (diff) |
Merge pull request #5956 from vespa-engine/balder/reapply-iterate-over-indexes-3
Iterate over indexes for ngram and quoting searcher.
4 files changed, 61 insertions, 82 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java index d4cad7f1246..5dcc533fb1f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/QuotingSearcher.java @@ -35,6 +35,7 @@ public class QuotingSearcher extends Searcher { } private static class QuoteTable { + private final int lowerUncachedBound; private final int upperUncachedBound; private final Map<Character, String> quoteMap; @@ -50,12 +51,10 @@ public class QuotingSearcher extends Searcher { boolean newIsEmpty = true; Map<Character, String> newQuoteMap = new HashMap<>(); for (Iterator<?> i = config.character().iterator(); i.hasNext(); ) { - QrQuotetableConfig.Character character - = (QrQuotetableConfig.Character)i.next(); + QrQuotetableConfig.Character character = (QrQuotetableConfig.Character)i.next(); if (character.ordinal() > 256) { newIsEmpty = false; - newQuoteMap.put(new Character((char)character.ordinal()), - character.quoting()); + newQuoteMap.put(new Character((char)character.ordinal()), character.quoting()); newUseMap = true; if (minOrd == 0 || character.ordinal() < minOrd) minOrd = character.ordinal(); @@ -64,8 +63,7 @@ public class QuotingSearcher extends Searcher { } else { newIsEmpty = false; - newLowerTable[character.ordinal()] - = character.quoting(); + newLowerTable[character.ordinal()] = character.quoting(); } } lowerUncachedBound = minOrd; @@ -75,22 +73,19 @@ public class QuotingSearcher extends Searcher { isEmpty = newIsEmpty; lowerTable = newLowerTable; } + public String get(char c) { - if (isEmpty) - return null; + if (isEmpty) return null; + int ord = (int)c; if (ord < 256) { return lowerTable[ord]; } else { - if ((!useMap) || ord < lowerUncachedBound - || ord > upperUncachedBound) - { + if ((!useMap) || ord < lowerUncachedBound || ord > upperUncachedBound) return null; - } - else { + else return quoteMap.get(new Character(c)); - } } } public boolean isEmpty() { @@ -107,35 +102,29 @@ public class QuotingSearcher extends Searcher { Result result = execution.search(query); execution.fill(result); QuoteTable translations = getQuoteTable(); - if (translations == null || translations.isEmpty()) { - return result; - } + if (translations == null || translations.isEmpty()) return result; + for (Iterator<Hit> i = result.hits().deepIterator(); i.hasNext(); ) { Hit h = i.next(); - if (h instanceof FastHit) { - quoteProperties((FastHit)h, translations); - } + if (h instanceof FastHit) + quoteFields((FastHit) h, translations); } return result; } - private void quoteProperties(FastHit hit, QuoteTable translations) { - for (Iterator<?> i = ((Set<?>) hit.fields().keySet()).iterator(); i.hasNext(); ) { - String propertyName = (String) i.next(); - Object entry = hit.getField(propertyName); - if (entry == null) { - continue; - } - Class<? extends Object> propertyType = entry.getClass(); - if (propertyType.equals(HitField.class)) { - quoteField((HitField) entry, translations); - } else if (propertyType.equals(String.class)) { - quoteProperty(hit, propertyName, (String)entry, translations); + private void quoteFields(FastHit hit, QuoteTable translations) { + hit.forEachField((fieldName, fieldValue) -> { + if (fieldValue != null) { + Class<?> fieldType = fieldValue.getClass(); + if (fieldType.equals(HitField.class)) + quoteField((HitField) fieldValue, translations); + else if (fieldType.equals(String.class)) + quoteField(hit, fieldName, (String) fieldValue, translations); } - } + }); } - private void quoteProperty(Hit hit, String fieldname, String toQuote, QuoteTable translations) { + private void quoteField(Hit hit, String fieldname, String toQuote, QuoteTable translations) { List<FieldPart> l = translate(toQuote, translations, true); if (l != null) { HitField hf = new HitField(fieldname, toQuote); @@ -144,13 +133,11 @@ public class QuotingSearcher extends Searcher { } } - private void quoteField(HitField field, QuoteTable translations) { for (ListIterator<FieldPart> i = field.listIterator(); i.hasNext(); ) { FieldPart f = i.next(); - if (!f.isFinal()) { - List<FieldPart> newFieldParts = translate(f.getContent(), translations, - f.isToken()); + if ( ! f.isFinal()) { + List<FieldPart> newFieldParts = translate(f.getContent(), translations, f.isToken()); if (newFieldParts != null) { i.remove(); for (Iterator<FieldPart> j = newFieldParts.iterator(); j.hasNext(); ) { @@ -161,33 +148,24 @@ public class QuotingSearcher extends Searcher { } } - private List<FieldPart> translate(String toQuote, QuoteTable translations, - boolean isToken) { + private List<FieldPart> translate(String toQuote, QuoteTable translations, boolean isToken) { List<FieldPart> newFieldParts = null; int lastIdx = 0; for (int i = 0; i < toQuote.length(); i++) { String quote = translations.get(toQuote.charAt(i)); if (quote != null) { - if (newFieldParts == null) { + if (newFieldParts == null) newFieldParts = new ArrayList<>(); - } - if (lastIdx != i) { - newFieldParts.add( - new StringFieldPart(toQuote.substring(lastIdx, i), - isToken)); - } + if (lastIdx != i) + newFieldParts.add(new StringFieldPart(toQuote.substring(lastIdx, i), isToken)); String initContent = Character.toString(toQuote.charAt(i)); - newFieldParts.add(new ImmutableFieldPart(initContent, - quote, - isToken)); + newFieldParts.add(new ImmutableFieldPart(initContent, quote, isToken)); lastIdx = i+1; } } - if (lastIdx > 0 && lastIdx < toQuote.length()) { - newFieldParts.add( - new StringFieldPart(toQuote.substring(lastIdx), - isToken)); - } + if (lastIdx > 0 && lastIdx < toQuote.length()) + newFieldParts.add(new StringFieldPart(toQuote.substring(lastIdx), isToken)); return newFieldParts; } + } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java index 2768a546cd0..399ff6194c8 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/NGramSearcher.java @@ -40,8 +40,8 @@ public class NGramSearcher extends Searcher { private final CharacterClasses characterClasses; public NGramSearcher(Linguistics linguistics) { - gramSplitter= linguistics.getGramSplitter(); - characterClasses= linguistics.getCharacterClasses(); + gramSplitter = linguistics.getGramSplitter(); + characterClasses = linguistics.getCharacterClasses(); } @Override @@ -54,7 +54,7 @@ public class NGramSearcher extends Searcher { if (rewritten) query.trace("Rewritten to n-gram matching",true,2); - Result result=execution.search(query); + Result result = execution.search(query); recombineNGrams(result.hits().deepIterator(), session); return result; } @@ -160,10 +160,11 @@ public class NGramSearcher extends Searcher { if (hit.isMeta()) continue; Object sddocname = hit.getField(Hit.SDDOCNAME_FIELD); if (sddocname == null) return; - for (String fieldName : hit.fieldKeys()) { // TODO: Iterate over indexes instead - Index index = session.getIndex(fieldName, sddocname.toString()); + for (Index index : session.getIndexes(sddocname.toString())) { if (index.isNGram() && (index.getHighlightSummary() || index.getDynamicSummary())) { - hit.setField(fieldName, recombineNGramsField(hit.getField(fieldName), index.getGramSize())); + Object fieldValue = hit.getField(index.getName()); + if (fieldValue != null) + hit.setField(index.getName(), recombineNGramsField(fieldValue, index.getGramSize())); } } } diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java index ee735104caa..691f877dfba 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java @@ -15,9 +15,9 @@ import com.yahoo.search.Result; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.hitfield.HitField; import com.yahoo.search.Searcher; -import com.yahoo.prelude.searcher.DocumentSourceSearcher; import com.yahoo.prelude.searcher.QuotingSearcher; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import org.junit.Test; import java.util.ArrayList; @@ -55,7 +55,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", "smith & jones"); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); assertEquals("smith & jones", check.hits().get(0).getField("title").toString()); assertTrue(check.hits().get(0).fields().containsKey("title")); @@ -75,7 +75,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", "&smith &jo& nes"); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); assertEquals("&smith &jo& nes", check.hits().get(0).getField("title").toString()); assertTrue(check.hits().get(0).fields().containsKey("title")); @@ -95,7 +95,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", new HitField("title", "&smith &jo& nes")); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); assertEquals("&smith &jo& nes", check.hits().get(0).getField("title").toString()); assertTrue(check.hits().get(0).fields().containsKey("title")); @@ -116,7 +116,7 @@ public class QuotingSearcherTestCase { hit.setRelevance(new Relevance(1)); hit.setField("title", Integer.valueOf(42)); r.hits().add(hit); - docsource.addResultSet(q, r); + docsource.addResult(q, r); Result check = doSearch(s, q, 0, 10, chained); // should not quote non-string properties assertEquals(Integer.valueOf(42), check.hits().get(0).getField("title")); diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java index 99b7eeff9a0..a3f7ff12319 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/test/NGramSearcherTestCase.java @@ -46,32 +46,32 @@ public class NGramSearcherTestCase { @Before public void setUp() { - searcher=new NGramSearcher(new SimpleLinguistics()); - indexFacts=new IndexFacts(); + searcher = new NGramSearcher(new SimpleLinguistics()); + indexFacts = new IndexFacts(); - Index defaultIndex=new Index("default"); - defaultIndex.setNGram(true,3); + Index defaultIndex = new Index("default"); + defaultIndex.setNGram(true, 3); defaultIndex.setDynamicSummary(true); - indexFacts.addIndex("default",defaultIndex); + indexFacts.addIndex("default", defaultIndex); - Index test=new Index("test"); + Index test = new Index("test"); test.setHighlightSummary(true); - indexFacts.addIndex("default",test); + indexFacts.addIndex("default", test); - Index gram2=new Index("gram2"); - gram2.setNGram(true,2); + Index gram2 = new Index("gram2"); + gram2.setNGram(true, 2); gram2.setDynamicSummary(true); - indexFacts.addIndex("default",gram2); + indexFacts.addIndex("default", gram2); - Index gram3=new Index("gram3"); - gram3.setNGram(true,3); + Index gram3 = new Index("gram3"); + gram3.setNGram(true, 3); gram3.setHighlightSummary(true); - indexFacts.addIndex("default",gram3); + indexFacts.addIndex("default", gram3); - Index gram14=new Index("gram14"); - gram14.setNGram(true,14); + Index gram14 = new Index("gram14"); + gram14.setNGram(true, 14); gram14.setDynamicSummary(true); - indexFacts.addIndex("default",gram14); + indexFacts.addIndex("default", gram14); } private IndexFacts getMixedSetup() { |