diff options
author | Jon Bratseth <bratseth@gmail.com> | 2024-05-16 05:32:30 -0600 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-05-16 05:32:30 -0600 |
commit | cced1ff316a6960a0a6d091f43b0eafd3ba2a78e (patch) | |
tree | be4954603c1b79329ccf59b5b6942ae1ffa5bbbe | |
parent | 9a3ef654bfe4dda2a3c0d74928a29d8e75e35ae9 (diff) | |
parent | 3c8369a8772f6f03193176b09bc10b8789c642a6 (diff) |
Merge pull request #31194 from vespa-engine/bratseth/stemming-trace
Trace no stemming due to language=UNKNOWN
5 files changed, 21 insertions, 20 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 15f3eece8a9..c39da176828 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -22,11 +22,6 @@ public class DispatchTuning { private final Integer maxHitsPerPartition; private final DispatchPolicy dispatchPolicy; private final Double minActiveDocsCoverage; - - public Double getTopkProbability() { - return topkProbability; - } - private final Double topkProbability; private DispatchTuning(Builder builder) { @@ -45,6 +40,8 @@ public class DispatchTuning { /** Returns the percentage of documents which must be available in a group for that group to receive queries */ public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; } + public Double getTopkProbability() { return topkProbability; } + public static class Builder { private Integer maxHitsPerPartition; @@ -71,14 +68,14 @@ public class DispatchTuning { } public static DispatchPolicy toDispatchPolicy(String policy) { - switch (policy.toLowerCase()) { - case "adaptive": case "random": return DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9 - case "round-robin": return DispatchPolicy.ROUNDROBIN; - case "latency-amortized-over-requests" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS; - case "latency-amortized-over-time" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME; - case "best-of-random-2" : return DispatchPolicy.BEST_OF_RANDOM_2; - default: throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'"); - } + return switch (policy.toLowerCase()) { + case "adaptive", "random" -> DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9 + case "round-robin" -> DispatchPolicy.ROUNDROBIN; + case "latency-amortized-over-requests" -> DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS; + case "latency-amortized-over-time" -> DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME; + case "best-of-random-2" -> DispatchPolicy.BEST_OF_RANDOM_2; + default -> throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'"); + }; } public Builder setMinActiveDocsCoverage(Double minCoverage) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index f27e9bbc281..0ea9cefc63e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -109,14 +109,17 @@ public class StemmingSearcher extends Searcher { private Item replaceTerms(Query q, IndexFacts.Session indexFacts) { Language language = q.getModel().getParsingLanguage(); - if (language == Language.UNKNOWN) return q.getModel().getQueryTree().getRoot(); + if (language == Language.UNKNOWN) { + q.trace("Language is unknown, not stemming", 3); + return q.getModel().getQueryTree().getRoot(); + } StemContext context = new StemContext(); context.isCJK = language.isCjk(); context.language = language; context.indexFacts = indexFacts; context.reverseConnectivity = createReverseConnectivities(q.getModel().getQueryTree().getRoot()); - q.trace("Stemming with language="+language, 3); + q.trace("Stemming with language " + language, 3); return scan(q.getModel().getQueryTree().getRoot(), context); } diff --git a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java index f6025dc6ba7..e3a559da8f9 100644 --- a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java @@ -30,9 +30,8 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; /** * Sets significance values on word items in the query tree. * - * @author MariusArhaug + * @author Marius Arhaug */ - @Provides(SignificanceSearcher.SIGNIFICANCE) @Before(STEMMING) public class SignificanceSearcher extends Searcher { @@ -112,6 +111,7 @@ public class SignificanceSearcher extends Searcher { public static double calculateIDF(long N, long nq_i) { return Math.log(1 + (N - nq_i + 0.5) / (nq_i + 0.5)); } + } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 91f5984481a..1ba10a31638 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -17,6 +17,7 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NumericInItem; +import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.PrefixItem; @@ -30,6 +31,7 @@ import com.yahoo.prelude.query.SuffixItem; import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WordAlternativesItem; import com.yahoo.prelude.query.WordItem; +import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.prelude.querytransform.QueryRewrite; import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; @@ -557,8 +559,7 @@ public class YqlParserTestCase { "title contains ({id: 1, connectivity: {\"id\": 3, weight: 7.0}}\"madonna\") " + "and title contains ({id: 2}\"saint\") " + "and title contains ({id: 3}\"angel\")"); - assertEquals("AND title:madonna title:saint title:angel", - parsed.toString()); + assertEquals("AND title:madonna title:saint title:angel", parsed.toString()); AndItem root = (AndItem) parsed.getRoot(); WordItem first = (WordItem) root.getItem(0); WordItem second = (WordItem) root.getItem(1); diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java index f8d0dc83abc..662d4a807c5 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java @@ -47,7 +47,7 @@ public class SimpleDetector implements Detector { } public Language guessLanguage(String input) { - if (input == null || input.length() == 0) { + if (input == null || input.isEmpty()) { return Language.UNKNOWN; } |