aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2024-05-16 05:32:30 -0600
committerGitHub <noreply@github.com>2024-05-16 05:32:30 -0600
commitcced1ff316a6960a0a6d091f43b0eafd3ba2a78e (patch)
treebe4954603c1b79329ccf59b5b6942ae1ffa5bbbe
parent9a3ef654bfe4dda2a3c0d74928a29d8e75e35ae9 (diff)
parent3c8369a8772f6f03193176b09bc10b8789c642a6 (diff)
Merge pull request #31194 from vespa-engine/bratseth/stemming-trace
Trace no stemming due to language=UNKNOWN
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java23
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java7
-rw-r--r--container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java4
-rw-r--r--container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java5
-rw-r--r--linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java2
5 files changed, 21 insertions, 20 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java
index 15f3eece8a9..c39da176828 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java
@@ -22,11 +22,6 @@ public class DispatchTuning {
private final Integer maxHitsPerPartition;
private final DispatchPolicy dispatchPolicy;
private final Double minActiveDocsCoverage;
-
- public Double getTopkProbability() {
- return topkProbability;
- }
-
private final Double topkProbability;
private DispatchTuning(Builder builder) {
@@ -45,6 +40,8 @@ public class DispatchTuning {
/** Returns the percentage of documents which must be available in a group for that group to receive queries */
public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; }
+ public Double getTopkProbability() { return topkProbability; }
+
public static class Builder {
private Integer maxHitsPerPartition;
@@ -71,14 +68,14 @@ public class DispatchTuning {
}
public static DispatchPolicy toDispatchPolicy(String policy) {
- switch (policy.toLowerCase()) {
- case "adaptive": case "random": return DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9
- case "round-robin": return DispatchPolicy.ROUNDROBIN;
- case "latency-amortized-over-requests" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS;
- case "latency-amortized-over-time" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME;
- case "best-of-random-2" : return DispatchPolicy.BEST_OF_RANDOM_2;
- default: throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'");
- }
+ return switch (policy.toLowerCase()) {
+ case "adaptive", "random" -> DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9
+ case "round-robin" -> DispatchPolicy.ROUNDROBIN;
+ case "latency-amortized-over-requests" -> DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS;
+ case "latency-amortized-over-time" -> DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME;
+ case "best-of-random-2" -> DispatchPolicy.BEST_OF_RANDOM_2;
+ default -> throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'");
+ };
}
public Builder setMinActiveDocsCoverage(Double minCoverage) {
diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
index f27e9bbc281..0ea9cefc63e 100644
--- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java
@@ -109,14 +109,17 @@ public class StemmingSearcher extends Searcher {
private Item replaceTerms(Query q, IndexFacts.Session indexFacts) {
Language language = q.getModel().getParsingLanguage();
- if (language == Language.UNKNOWN) return q.getModel().getQueryTree().getRoot();
+ if (language == Language.UNKNOWN) {
+ q.trace("Language is unknown, not stemming", 3);
+ return q.getModel().getQueryTree().getRoot();
+ }
StemContext context = new StemContext();
context.isCJK = language.isCjk();
context.language = language;
context.indexFacts = indexFacts;
context.reverseConnectivity = createReverseConnectivities(q.getModel().getQueryTree().getRoot());
- q.trace("Stemming with language="+language, 3);
+ q.trace("Stemming with language " + language, 3);
return scan(q.getModel().getQueryTree().getRoot(), context);
}
diff --git a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java
index f6025dc6ba7..e3a559da8f9 100644
--- a/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java
+++ b/container-search/src/main/java/com/yahoo/search/significance/SignificanceSearcher.java
@@ -30,9 +30,8 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING;
/**
* Sets significance values on word items in the query tree.
*
- * @author MariusArhaug
+ * @author Marius Arhaug
*/
-
@Provides(SignificanceSearcher.SIGNIFICANCE)
@Before(STEMMING)
public class SignificanceSearcher extends Searcher {
@@ -112,6 +111,7 @@ public class SignificanceSearcher extends Searcher {
public static double calculateIDF(long N, long nq_i) {
return Math.log(1 + (N - nq_i + 0.5) / (nq_i + 0.5));
}
+
}
diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
index 91f5984481a..1ba10a31638 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
@@ -17,6 +17,7 @@ import com.yahoo.prelude.query.Item;
import com.yahoo.prelude.query.MarkerWordItem;
import com.yahoo.prelude.query.NearestNeighborItem;
import com.yahoo.prelude.query.NumericInItem;
+import com.yahoo.prelude.query.OrItem;
import com.yahoo.prelude.query.PhraseItem;
import com.yahoo.prelude.query.PhraseSegmentItem;
import com.yahoo.prelude.query.PrefixItem;
@@ -30,6 +31,7 @@ import com.yahoo.prelude.query.SuffixItem;
import com.yahoo.prelude.query.WeakAndItem;
import com.yahoo.prelude.query.WordAlternativesItem;
import com.yahoo.prelude.query.WordItem;
+import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation;
import com.yahoo.prelude.querytransform.QueryRewrite;
import com.yahoo.processing.IllegalInputException;
import com.yahoo.search.Query;
@@ -557,8 +559,7 @@ public class YqlParserTestCase {
"title contains ({id: 1, connectivity: {\"id\": 3, weight: 7.0}}\"madonna\") " +
"and title contains ({id: 2}\"saint\") " +
"and title contains ({id: 3}\"angel\")");
- assertEquals("AND title:madonna title:saint title:angel",
- parsed.toString());
+ assertEquals("AND title:madonna title:saint title:angel", parsed.toString());
AndItem root = (AndItem) parsed.getRoot();
WordItem first = (WordItem) root.getItem(0);
WordItem second = (WordItem) root.getItem(1);
diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
index f8d0dc83abc..662d4a807c5 100644
--- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
+++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleDetector.java
@@ -47,7 +47,7 @@ public class SimpleDetector implements Detector {
}
public Language guessLanguage(String input) {
- if (input == null || input.length() == 0) {
+ if (input == null || input.isEmpty()) {
return Language.UNKNOWN;
}