diff options
Diffstat (limited to 'container-search')
81 files changed, 1279 insertions, 590 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index b320a1090ae..8cd32985057 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -503,7 +503,8 @@ "public void <init>(com.yahoo.prelude.query.Item, java.util.Collection)", "public com.yahoo.prelude.query.Item$ItemType getItemType()", "public java.lang.String getName()", - "protected void adding(com.yahoo.prelude.query.Item)" + "protected void adding(com.yahoo.prelude.query.Item)", + "public static boolean acceptsChildrenOfType(com.yahoo.prelude.query.Item$ItemType)" ], "fields": [] }, @@ -719,7 +720,6 @@ "public static final enum com.yahoo.prelude.query.Item$ItemType WORD", "public static final enum com.yahoo.prelude.query.Item$ItemType INT", "public static final enum com.yahoo.prelude.query.Item$ItemType PHRASE", - "public static final enum com.yahoo.prelude.query.Item$ItemType PAREN", "public static final enum com.yahoo.prelude.query.Item$ItemType PREFIX", "public static final enum com.yahoo.prelude.query.Item$ItemType SUBSTRING", "public static final enum com.yahoo.prelude.query.Item$ItemType NEAR", @@ -1594,6 +1594,7 @@ "public boolean hasItemClass(java.lang.Class)", "public com.yahoo.prelude.query.Item createItemClass()", "public java.lang.String toSign()", + "public com.yahoo.prelude.query.Item$ItemType toItemType()", "public boolean equals(java.lang.Object)", "public int hashCode()", "public java.lang.String toString()" @@ -2118,6 +2119,7 @@ "public void <init>(com.yahoo.component.ComponentId, java.util.List, boolean)", "public void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean)", "protected void <init>(com.yahoo.component.ComponentId, java.util.List, com.yahoo.search.cluster.Hasher, boolean, boolean)", + "public java.lang.String name()", "public final void ping(com.yahoo.search.cluster.ClusterMonitor, java.lang.Object, java.util.concurrent.Executor)", "protected abstract com.yahoo.prelude.Pong ping(com.yahoo.prelude.Ping, java.lang.Object)", "protected java.lang.Object getFirstConnection(com.yahoo.search.cluster.Hasher$NodeList, int, int, com.yahoo.search.Query)", @@ -2211,6 +2213,7 @@ "abstract" ], "methods": [ + "public java.lang.String name()", "public abstract void working(java.lang.Object)", "public abstract void failed(java.lang.Object)", "public void ping(java.lang.Object, java.util.concurrent.Executor)", @@ -5234,13 +5237,16 @@ "public java.lang.String getFormat()", "public void setFormat(java.lang.String)", "public java.lang.Object clone()", - "public boolean equals(java.lang.Object)", - "public int hashCode()", "public boolean getTiming()", "public void setTiming(boolean)", "public java.util.Set getSummaryFields()", + "public void setSummaryFields(java.lang.String)", + "public boolean getTensorShortForm()", + "public void setTensorShortForm(java.lang.String)", + "public void setTensorShortForm(boolean)", "public void prepare()", - "public void setSummaryFields(java.lang.String)" + "public boolean equals(java.lang.Object)", + "public int hashCode()" ], "fields": [ "public static final java.lang.String PRESENTATION", @@ -5248,6 +5254,7 @@ "public static final java.lang.String TIMING", "public static final java.lang.String SUMMARY", "public static final java.lang.String SUMMARY_FIELDS", + "public static final java.lang.String TENSORS", "public static final java.lang.String FORMAT" ] }, @@ -5292,6 +5299,7 @@ "public boolean isEmpty()", "public com.yahoo.prelude.query.Item and(com.yahoo.prelude.query.Item)", "public static java.util.List getPositiveTerms(com.yahoo.prelude.query.Item)", + "public int treeSize()", "public bridge synthetic com.yahoo.prelude.query.CompositeItem clone()", "public bridge synthetic com.yahoo.prelude.query.Item clone()", "public bridge synthetic java.lang.Object clone()" @@ -6241,7 +6249,7 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.search.query.profile.config.QueryProfilesConfig)", + "public void <init>(com.yahoo.search.query.profile.config.QueryProfilesConfig, java.util.concurrent.Executor)", "public void <init>()", "public void <init>(com.yahoo.search.query.profile.types.QueryProfileTypeRegistry)", "public final void register(com.yahoo.search.query.profile.compiled.CompiledQueryProfile)", @@ -7154,6 +7162,7 @@ "methods": [ "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean)", "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean, boolean)", + "protected void <init>(boolean, boolean, boolean)", "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean, boolean, boolean)", "public void accept(java.lang.String, java.lang.Object)", "public void accept(java.lang.String, byte[], int, int)", @@ -7195,6 +7204,7 @@ "public void endResponse()", "public java.lang.String getEncoding()", "public java.lang.String getMimeType()", + "protected com.yahoo.search.rendering.JsonRenderer$FieldConsumer createFieldConsumer(boolean)", "protected com.yahoo.search.rendering.JsonRenderer$FieldConsumer createFieldConsumer(com.fasterxml.jackson.core.JsonGenerator, boolean)" ], "fields": [] diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 51c82eac264..ae953426e63 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -348,7 +348,7 @@ public class ClusterSearcher extends Searcher { if (query.getOffset() > 0 || query.getHits() < mergedResult.hits().size()) { if (mergedResult.getHitOrderer() != null) { // Make sure we have the necessary data for sorting - searcher.fill(mergedResult, Execution.ATTRIBUTEPREFETCH, execution); + searcher.fill(mergedResult, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS, execution); } mergedResult.hits().trim(query.getOffset(), query.getHits()); query.setOffset(0); // Needed when doing a trim diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java index df87de2a12b..1e0cfa3be9e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java @@ -51,12 +51,17 @@ public final class DocsumDefinitionSet { } if (ds == null) { throw new ConfigurationException("Fetched hit with summary class " + summaryClass + - ", but this summary class is not in current summary config (" + toString() + ")" + + ", but this summary class is not in current summary config (" + this + ")" + " (that is, you asked for something unknown, and no default was found)"); } return ds; } + /** Do we have a summary definition with the given name */ + public boolean hasDocsum(String summaryClass) { + return definitionsByName.containsKey(summaryClass); + } + /** * Makes data available for decoding for the given hit. * @@ -66,7 +71,7 @@ public final class DocsumDefinitionSet { * @return Error message or null on success. * @throws ConfigurationException if the summary class of this hit is missing */ - public final String lazyDecode(String summaryClass, byte[] data, FastHit hit) { + public String lazyDecode(String summaryClass, byte[] data, FastHit hit) { ByteBuffer buffer = ByteBuffer.wrap(data); buffer.order(ByteOrder.LITTLE_ENDIAN); long docsumClassId = buffer.getInt(); @@ -83,6 +88,7 @@ public final class DocsumDefinitionSet { return null; } + @Override public String toString() { StringBuilder sb = new StringBuilder(); for (Map.Entry<String, DocsumDefinition> e : definitionsByName.entrySet() ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 7c7f01b414a..094367dc140 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -377,7 +377,7 @@ public class FastHit extends Hit { } } - private static void appendAsHex(byte [] gid, StringBuilder sb) { + private static void appendAsHex(byte[] gid, StringBuilder sb) { for (byte b : gid) { String hex = Integer.toHexString(0xFF & b); if (hex.length() == 1) { diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java index 1a92e388f47..7abaf99c93d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/VespaBackEndSearcher.java @@ -37,6 +37,9 @@ import java.util.logging.Logger; */ public abstract class VespaBackEndSearcher extends PingableSearcher { + /** for vespa-internal use only; consider renaming the summary class */ + public static final String SORTABLE_ATTRIBUTES_SUMMARY_CLASS = "attributeprefetch"; + private static final CompoundName TRACE_DISABLE = new CompoundName("trace.disable"); private String serverId; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java index 55adc3cba8d..0c4dff15744 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeIndexedItem.java @@ -22,9 +22,7 @@ public abstract class CompositeIndexedItem extends CompositeTaggableItem impleme private String index = ""; - /** - * The name of the index this belongs to, or "" (never null) if not specified - **/ + /** The name of the index this belongs to, or "" (never null) if not specified */ public String getIndexName() { return index; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java b/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java index 97cbfac0b2d..500c68cf4bd 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/EquivItem.java @@ -83,11 +83,16 @@ public class EquivItem extends CompositeTaggableItem { super.adding(item); Validator.ensure("Could not add an item of type " + item.getItemType() + ": Equiv can only have word, wordAlternatives, int, exact, or phrase as children", - item.getItemType() == ItemType.WORD || - item.getItemType() == ItemType.WORD_ALTERNATIVES || - item.getItemType() == ItemType.INT || - item.getItemType() == ItemType.EXACT || - item.getItemType() == ItemType.PHRASE); + acceptsChildrenOfType(item.getItemType())); + } + + /** Returns true if this accepts child items of the given type */ + public static boolean acceptsChildrenOfType(ItemType itemType) { + return itemType == ItemType.WORD || + itemType == ItemType.WORD_ALTERNATIVES || + itemType == ItemType.INT || + itemType == ItemType.EXACT || + itemType == ItemType.PHRASE; } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index 670983f0cd7..41d897b8d83 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -39,7 +39,7 @@ public abstract class Item implements Cloneable { WORD(4), INT(5), PHRASE(6), - PAREN(7), // TODO not used - remove on Vespa 8 + // 7 was PAREN, unused in Vespa 7 PREFIX(8), SUBSTRING(9), NEAR(11), diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java index 8c4c5c84a28..a93dd1b9de4 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.query.QueryTree; @@ -19,6 +20,8 @@ public class QueryCanonicalizer { /** The name of the operation performed by this, for use in search chain ordering */ public static final String queryCanonicalization = "queryCanonicalization"; + private static final CompoundName MAX_QUERY_ITEMS = new CompoundName("maxQueryItems"); + /** * Validates this query and carries out possible operations on this query * which simplifies it without changing its semantics. @@ -26,7 +29,17 @@ public class QueryCanonicalizer { * @return null if the query is valid, an error message if it is invalid */ public static String canonicalize(Query query) { - return canonicalize(query.getModel().getQueryTree()); + Integer maxQueryItems = query.properties().getInteger(MAX_QUERY_ITEMS, Integer.MAX_VALUE); + return canonicalize(query.getModel().getQueryTree(), maxQueryItems); + } + + /** + * Canonicalizes this query, allowing any query tree size + * + * @return null if the query is valid, an error message if it is invalid + */ + public static String canonicalize(QueryTree queryTree) { + return canonicalize(queryTree, Integer.MAX_VALUE); } /** @@ -34,10 +47,12 @@ public class QueryCanonicalizer { * * @return null if the query is valid, an error message if it is invalid */ - public static String canonicalize(QueryTree query) { + private static String canonicalize(QueryTree query, Integer maxQueryItems) { ListIterator<Item> rootItemIterator = query.getItemIterator(); CanonicalizationResult result = recursivelyCanonicalize(rootItemIterator.next(), rootItemIterator); if (query.isEmpty() && ! result.isError()) result = CanonicalizationResult.error("No query"); + int itemCount = query.treeSize(); + if (itemCount > maxQueryItems) result = CanonicalizationResult.error(String.format("Query tree exceeds allowed item count. Configured limit: %d - Item count: %d", maxQueryItems, itemCount)); return result.error().orElse(null); // preserve old API, unfortunately } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java index 309befd80f5..fbbc746b130 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/TermType.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/TermType.java @@ -3,35 +3,41 @@ package com.yahoo.prelude.query; /** - * A term type enumeration + * A term type enumeration. * * @author bratseth * @author Steinar Knutsen */ public class TermType { - public static final TermType RANK = new TermType("rank", RankItem.class, null, "$"); + public static final TermType RANK = new TermType("rank", Item.ItemType.RANK, RankItem.class, null, "$"); - public static final TermType AND = new TermType("and", AndItem.class, null, "+"); + public static final TermType AND = new TermType("and", Item.ItemType.AND, AndItem.class, null, "+"); - public static final TermType OR = new TermType("or", OrItem.class, null, "?"); + public static final TermType OR = new TermType("or", Item.ItemType.OR, OrItem.class, null, "?"); - public static final TermType NOT = new TermType("not", NotItem.class, null, "-"); + public static final TermType NOT = new TermType("not", Item.ItemType.NOT, NotItem.class, null, "-"); - public static final TermType PHRASE = new TermType("phrase", PhraseItem.class, null, "\""); + public static final TermType PHRASE = new TermType("phrase", Item.ItemType.PHRASE, PhraseItem.class, null, "\""); - public static final TermType EQUIV = new TermType("equiv", EquivItem.class, null, ""); + public static final TermType EQUIV = new TermType("equiv", Item.ItemType.EQUIV, EquivItem.class, null, ""); - public static final TermType DEFAULT = new TermType("", CompositeItem.class, AndItem.class, ""); + public static final TermType DEFAULT = new TermType("", Item.ItemType.AND, CompositeItem.class, AndItem.class, ""); public final String name; + private final Item.ItemType itemType; private final String sign; private final Class<? extends CompositeItem> instanceClass; private final Class<? extends CompositeItem> itemClass; - private TermType(String name, Class<? extends CompositeItem> itemClass, Class<? extends CompositeItem> instanceClass, String sign) { + private TermType(String name, + Item.ItemType itemType, + Class<? extends CompositeItem> itemClass, + Class<? extends CompositeItem> instanceClass, + String sign) { this.name = name; + this.itemType = itemType; this.itemClass = itemClass; if (instanceClass == null) { this.instanceClass = itemClass; @@ -73,6 +79,8 @@ public class TermType { return sign; } + public Item.ItemType toItemType() { return itemType; } + @Override public boolean equals(Object o) { if ( ! (o instanceof TermType)) return false; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 5fb90deaa0a..586d1d32d57 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -3,7 +3,6 @@ package com.yahoo.prelude.query.parser; import com.yahoo.language.Language; import com.yahoo.language.process.Segmenter; -import com.yahoo.log.event.*; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.*; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java index 932f4ea7c38..5dcd591b1fc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/BlendingSearcher.java @@ -8,6 +8,7 @@ import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.component.chain.dependencies.Provides; import com.yahoo.container.QrSearchersConfig; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; @@ -113,7 +114,8 @@ public class BlendingSearcher extends Searcher { private Result sortAndTrimResults(Result result, Query q, int offset, int hits, Execution execution) { if (q.getRanking().getSorting() != null) { - execution.fillAttributes(result); // Always correct as we can only sort on attributes + // TODO: remove or rename this internal summary class for Vespa 9 + execution.fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); result.hits().sort(); } result.hits().trim(offset, hits); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java index 8e137d99951..4980b035876 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java @@ -1,10 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics; -import com.yahoo.language.Language; import com.yahoo.language.Linguistics; -import com.yahoo.language.process.StemMode; -import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.semantics.rule.CompositeCondition; import com.yahoo.prelude.semantics.rule.Condition; import com.yahoo.prelude.semantics.rule.NamedCondition; @@ -76,12 +73,9 @@ public class RuleBase { */ private boolean usesAutomata = false; - private RuleBaseLinguistics linguistics; - /** Creates an empty rule base */ - public RuleBase(String name, Linguistics linguistics) { + public RuleBase(String name) { this.name = name; - this.linguistics = new RuleBaseLinguistics(StemMode.BEST, Language.ENGLISH, linguistics); } /** @@ -284,7 +278,7 @@ public class RuleBase { } /** - * Set to truew if this uses an automata, even if an automata is not present right now. + * Set to true if this uses an automata, even if an automata is not present right now. * Useful to validate without having automatas available */ void setUsesAutomata(boolean usesAutomata) { this.usesAutomata = usesAutomata; } @@ -342,7 +336,7 @@ public class RuleBase { } // TODO: Values are not added right now - protected void annotatePhrase(PhraseMatcher.Phrase phrase,Query query,int traceLevel) { + protected void annotatePhrase(PhraseMatcher.Phrase phrase, Query query, int traceLevel) { for (StringTokenizer tokens = new StringTokenizer(phrase.getData(), "|", false); tokens.hasMoreTokens(); ) { String token = tokens.nextToken(); int semicolonIndex = token.indexOf(";"); @@ -357,12 +351,12 @@ public class RuleBase { phrase.getItem(0).addAnnotation(annotation, phrase); if (traceLevel >= 4) query.trace(" Annotating '" + phrase + "' as " + annotation + - (value.equals("") ? "" :"=" + value),false,1); + (value.equals("") ? "" :"=" + value),false,1); } } private void makeReferences() { - for (Iterator<ProductionRule> i=ruleIterator(); i.hasNext(); ) { + for (Iterator<ProductionRule> i = ruleIterator(); i.hasNext(); ) { ProductionRule rule = i.next(); rule.makeReferences(this); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java index acbf9a7ffb6..6e7286cb8dc 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java @@ -99,7 +99,7 @@ public class RuleImporter { reader = IOUtils.createReader(fileName, "utf-8"); File file = new File(fileName); String absoluteFileName = file.getAbsolutePath(); - var ruleBase = new RuleBase(stripLastName(file.getName()), linguistics); + var ruleBase = new RuleBase(stripLastName(file.getName())); privateImportFromReader(reader, absoluteFileName, automataFile, ruleBase); return ruleBase; } @@ -205,7 +205,7 @@ public class RuleImporter { /** Imports an unitialized rule base */ public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws ParseException { if (config == null) throw new IllegalStateException("Must initialize with config if importing from config"); - RuleBase ruleBase = new RuleBase(ruleBaseConfig.name(), linguistics); + RuleBase ruleBase = new RuleBase(ruleBaseConfig.name()); return privateImportFromReader(new StringReader(ruleBaseConfig.rules()), "semantic-rules.cfg", ruleBaseConfig.automata(),ruleBase); @@ -234,7 +234,7 @@ public class RuleImporter { public RuleBase privateImportFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { try { if (ruleBase == null) - ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName, linguistics); + ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName); ruleBase.setSource(sourceName.replace('\\', '/')); new SemanticsParser(reader, linguistics).semanticRules(ruleBase, this); if (automataFile != null && !automataFile.isEmpty()) diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java index 85bcdf8bd20..d5ed89b0724 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Evaluation.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.semantics.engine; import com.yahoo.prelude.query.*; +import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.search.Query; import com.yahoo.search.query.QueryTree; @@ -32,11 +33,13 @@ public class Evaluation { /** The rule evaluation context, can be reset once the rule is evaluated */ private final RuleEvaluation ruleEvaluation; + private final RuleBase ruleBase; + /** * The amount of context information to collect about this evaluation. * 0 means no context information, higher numbers means more context information. */ - private int traceLevel = 0; + private final int traceLevel; private String traceIndentation = ""; @@ -49,8 +52,8 @@ public class Evaluation { /** Should we allow stemmed matches? */ private boolean stemming = true; - public Evaluation(Query query) { - this(query,0); + public Evaluation(Query query, RuleBase ruleBase) { + this(query, ruleBase, 0); } /** @@ -59,13 +62,17 @@ public class Evaluation { * @param query the query this evaluation is for * @param traceLevel the amount of tracing to do */ - public Evaluation(Query query, int traceLevel) { + public Evaluation(Query query, RuleBase ruleBase, int traceLevel) { this.query = query; + this.ruleBase = ruleBase; this.traceLevel = traceLevel; reset(); ruleEvaluation = new RuleEvaluation(this); } + /** Returns the rule base this evaluates over */ + public RuleBase ruleBase() { return ruleBase; } + /** Resets the item iterator to point to the first item */ public void reset() { if (flattenedItems != null) @@ -134,11 +141,11 @@ public class Evaluation { /** Adds an item to the query being evaluated in a way consistent with the query type */ // TODO: Add this functionality to Query? public void addItem(Item item, TermType termType) { - Item root= query.getModel().getQueryTree().getRoot(); - if (root==null) + Item root = query.getModel().getQueryTree().getRoot(); + if (root == null) query.getModel().getQueryTree().setRoot(item); else - query.getModel().getQueryTree().setRoot(combineItems(root,item,termType)); + query.getModel().getQueryTree().setRoot(combineItems(root, item, termType)); } /** Removes this item */ @@ -151,18 +158,18 @@ public class Evaluation { * equal items */ public void removeItemByIdentity(Item item) { - int position=findIndexByIdentity(item); - if (position>=0) + int position = findIndexByIdentity(item); + if (position >= 0) item.getParent().removeItem(position); else item.getParent().removeItem(item); // Fallback to removeField by equal() } private int findIndexByIdentity(Item item) { - int position=0; - for (Iterator<Item> i=item.getParent().getItemIterator(); i.hasNext(); ) { - Item child=i.next(); - if (item==child) { + int position = 0; + for (Iterator<Item> i = item.getParent().getItemIterator(); i.hasNext(); ) { + Item child = i.next(); + if (item == child) { return position; } position++; @@ -172,7 +179,7 @@ public class Evaluation { /** Removes an item, prefers the one at/close to the given position if there are multiple ones */ public void removeItem(int position,Item item) { - Item removeCandidate=item.getParent().getItem(position); + Item removeCandidate = item.getParent().getItem(position); if (removeCandidate.equals(item)) // Remove based on position item.getParent().removeItem(position); else @@ -189,7 +196,7 @@ public class Evaluation { if (!(item instanceof SegmentItem)) { return item; } - CompositeItem converted = null; + CompositeItem converted; if (item instanceof AndSegmentItem) { converted = new AndItem(); } else if (item instanceof PhraseSegmentItem) { @@ -238,16 +245,21 @@ public class Evaluation { /** * Inserts an item to the query being evaluated in a way consistent with the query type * - * @param item the item to insert - * @param parent the parent of this item, or null to set the root - * @param index the index at which to insert this into the parent - * @param desiredParentType the desired type of the composite which contains item when this returns + * @param items the items to insert + * @param parent the parent of these items, or null to set the root + * @param index the index at which to insert these into the parent + * @param desiredParentType the desired type of the composite which contains items when this returns */ - public void insertItem(Item item, CompositeItem parent, int index, TermType desiredParentType) { + public void insertItems(List<Item> items, CompositeItem parent, int index, TermType desiredParentType) { if (isEmpty(parent)) { - CompositeItem newParent = (CompositeItem)desiredParentType.createItemClass(); - newParent.addItem(item); - query.getModel().getQueryTree().setRoot(newParent); + if (items.size() == 1 && desiredParentType.hasItemClass(items.get(0).getClass())) { + query.getModel().getQueryTree().setRoot(items.get(0)); + } + else { + CompositeItem newParent = (CompositeItem) desiredParentType.createItemClass(); + items.forEach(item -> newParent.addItem(item)); + query.getModel().getQueryTree().setRoot(newParent); + } return; } @@ -260,34 +272,37 @@ public class Evaluation { } if (( desiredParentType == TermType.DEFAULT || desiredParentType.hasItemClass(parent.getClass()) ) - && equalIndexNameIfParentIsPhrase(item, parent)) { - addItem(parent, index, item, desiredParentType); + && equalIndexNameIfParentIsPhrase(items, parent)) { + for (Item item : items) + addItem(parent, index, item, desiredParentType); } - else if (incompatible(desiredParentType, parent)) { - insertIncompatibleItem(item, parent, query, desiredParentType); + else if (parent.items().isEmpty()) { + CompositeItem parentsParent = parent.getParent(); + CompositeItem newParent = newParent(desiredParentType); + items.forEach(item -> newParent.addItem(item)); + parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); + } + else if (items.size() == 1 && desiredParentType.hasItemClass(items.get(0).getClass())) { + addItem(parent, index, items.get(0), desiredParentType); } else { - insertIncompatibleItemAsParent(item, parent, query, desiredParentType); + insertWithDesiredParentType(items, parent, desiredParentType); } } + /** Returns true if this item represents an empty query *tree*. */ private boolean isEmpty(Item item) { if (item == null) return true; if (item instanceof QueryTree && ((QueryTree) item).isEmpty()) return true; return false; } - /** Returns true if the desired type cannot have childCandidate as a child */ - private boolean incompatible(TermType desiredParentType, Item childCandidate) { - return desiredParentType == TermType.EQUIV && childCandidate.getItemType() != Item.ItemType.EQUIV; - } - private void addItem(CompositeItem parent, int index, Item item, TermType desiredParentType) { if (parent instanceof NotItem) { if (index == 0 && parent.getItem(0) == null) { // Case 1: The current positive is null and we are adding a positive parent.setItem(0, item); } - else if (index<=1 && !(parent.getItem(0) instanceof CompositeItem)) { // Case 2: The positive must become a composite + else if (index <= 1 && !(parent.getItem(0) instanceof CompositeItem)) { // Case 2: The positive must become a composite CompositeItem positiveComposite = (CompositeItem)desiredParentType.createItemClass(); positiveComposite.addItem(parent.getItem(0)); positiveComposite.addItem(index, item); @@ -313,50 +328,45 @@ public class Evaluation { } /** A special purpose check used to simplify the above */ - private boolean equalIndexNameIfParentIsPhrase(Item item,CompositeItem parent) { + private boolean equalIndexNameIfParentIsPhrase(List<Item> items, CompositeItem parent) { if ( ! (parent instanceof PhraseItem)) return true; - if ( ! (item instanceof IndexedItem)) return true; - - return ((PhraseItem)parent).getIndexName().equals(((IndexedItem)item).getIndexName()); - } - - private void insertIncompatibleItem(Item item, CompositeItem parent, Query query, TermType desiredParentType) { - CompositeItem newParent; - if (desiredParentType == TermType.DEFAULT) - newParent = new AndItem(); - else - newParent = (CompositeItem)desiredParentType.createItemClass(); + var phrase = (PhraseItem)parent; - newParent.addItem(item); - parent.addItem(newParent); + for (Item item : items) { + if ( ! (item instanceof IndexedItem)) continue; + var indexedItem = (IndexedItem)item; + if (! indexedItem.getIndexName().equals(phrase.getIndexName())) return false; + } + return true; } - private void insertIncompatibleItemAsParent(Item item, CompositeItem parent, Query query, TermType desiredParentType) { - // Create new parent - CompositeItem newParent; - if (desiredParentType == TermType.DEFAULT) - newParent = new AndItem(); - else - newParent = (CompositeItem)desiredParentType.createItemClass(); - - // Save previous parent parent + private void insertWithDesiredParentType(List<Item> items, CompositeItem parent, TermType desiredParentType) { CompositeItem parentsParent = parent.getParent(); - // Add items to new parent - newParent.addItem(parent); - newParent.addItem(item); + CompositeItem newParent = newParent(desiredParentType); + + if (! (parentsParent instanceof QueryTree) && parentsParent.getItemType() == newParent.getItemType()) { // Collapse + newParent = parentsParent; + } - // Insert new parent as root or child of old parents parent - if (parentsParent == null) { - query.getModel().getQueryTree().setRoot(newParent); + for (Item item : items) + newParent.addItem(item); + if (desiredParentType == TermType.EQUIV || desiredParentType == TermType.PHRASE) { // insert new parent below the current + parent.addItem(newParent); } - else { - parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); + else { // insert new parent above the current + newParent.addItem(parent); + if (newParent != parentsParent) // Insert new parent as root or child of old parent's parent + parentsParent.setItem(parentsParent.getItemIndex(parent), newParent); } } - private Item combineItems(Item first,Item second,TermType termType) { + private CompositeItem newParent(TermType desiredParentType) { + return desiredParentType == TermType.DEFAULT ? new AndItem() : (CompositeItem)desiredParentType.createItemClass(); + } + + private Item combineItems(Item first, Item second, TermType termType) { if (first instanceof NullItem) { return second; } else if (first instanceof NotItem) { @@ -378,6 +388,10 @@ public class Evaluation { return composite; } else { + if (combined instanceof EquivItem) { + first = makeEquivCompatible(first); + second = makeEquivCompatible(second); + } combined.addItem(first); combined.addItem(second); // Also works for nots return combined; @@ -394,6 +408,22 @@ public class Evaluation { } } + private Item makeEquivCompatible(Item item) { + if (item instanceof AndItem || item instanceof WeakAndItem) { + PhraseItem phrase = new PhraseItem(); + List<Item> children = ((CompositeItem)item).items(); + if (children.isEmpty()) return phrase; + String index = ((IndexedItem)children.get(0)).getIndexName(); + for (var child : ((CompositeItem)item).items()) + phrase.addItem(child); + phrase.setIndexName(index); + return phrase; + } + else { + return item; // Compatible, or can't be made so + } + } + private CompositeItem createType(TermType termType) { if (termType == TermType.DEFAULT) { if (query.getModel().getType() == Query.Type.ANY) diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java index 6098fbee327..6ed09926ccb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/Match.java @@ -60,7 +60,12 @@ public class Match { /** Returns a new item representing this match */ public Item toItem(String label) { - var newItem = new WordItem(getReplaceValue(), label); + return toItem(label, getReplaceValue()); + } + + /** Returns a new item representing this match */ + public Item toItem(String label, String term) { + var newItem = new WordItem(term, label); newItem.setWeight(item.getWeight()); return newItem; } @@ -81,4 +86,9 @@ public class Match { return true; } + @Override + public String toString() { + return "match of " + item + " at " + position + " to be replaced by " + replaceValue; + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java index 605f3a23e10..b7083414e85 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/ReferencedMatches.java @@ -8,7 +8,7 @@ import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PhraseItem; /** - * The Matches referenced by a particular context name in a rule evaluation + * The matches referenced by a particular context name in a rule evaluation * * @author bratseth */ diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java index dd6610d1184..e2756afbb2e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java @@ -37,7 +37,7 @@ public class RuleEngine { // Probably create indices on the first term like Prolog implementations use to boolean matchedAnything = false; - Evaluation evaluation = new Evaluation(query, traceLevel); + Evaluation evaluation = new Evaluation(query, rules, traceLevel); if (traceLevel >= 2) evaluation.trace(2,"Evaluating query '" + evaluation.getQuery().getModel().getQueryTree().getRoot() + "':"); for (ListIterator<ProductionRule> i = rules.ruleIterator(); i.hasNext(); ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java index 2a00843a85c..1da16b4d277 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEvaluation.java @@ -190,8 +190,9 @@ public class RuleEvaluation { // TODO: Simplistic yet. Nedd to support context nesting etc. public void entering(String context) { if (context == null) return; - if (matchReferences != null && matchReferences.contains(context)) + if (matchReferences != null && matchReferences.contains(context)) { currentContext = context; + } } public void leaving(String context) { @@ -209,7 +210,7 @@ public class RuleEvaluation { */ public void addMatch(FlattenedItem item, String replaceString) { evaluation.makeParentMutable(item.getItem()); - Match match = new Match(item,replaceString); + Match match = new Match(item, replaceString); if (currentContext != null) { ReferencedMatches matches = getReferencedMatches(currentContext); if (matches == null) { @@ -241,8 +242,8 @@ public class RuleEvaluation { public Evaluation getEvaluation() { return evaluation; } /** Adds an item to the query being evaluated in a way consistent with the query type */ - public void addItem(Item item, TermType termType) { - evaluation.addItem(item,termType); + public void addItems(List<Item> items, TermType termType) { + items.forEach(item -> evaluation.addItem(item, termType)); } public void removeItem(Item item) { @@ -262,13 +263,13 @@ public class RuleEvaluation { /** * Inserts an item to the query being evaluated in a way consistent with the query type * - * @param item the item to insert + * @param items the items to insert * @param parent the parent of this item, or null to set the root * @param index the index at which to insert this into the parent * @param termType the kind of item to index, this decides the resulting structure */ - public void insertItem(Item item, CompositeItem parent, int index, TermType termType) { - evaluation.insertItem(item, parent, index, termType); + public void insertItems(List<Item> items, CompositeItem parent, int index, TermType termType) { + evaluation.insertItems(items, parent, index, termType); } /** Returns a read-only view of the items of this */ diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java index 77bffa5778e..a49272a73ea 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/CompositeCondition.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; +import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -14,28 +15,28 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; */ public abstract class CompositeCondition extends Condition { - private List<Condition> conditions=new java.util.ArrayList<>(); + private final List<Condition> conditions = new java.util.ArrayList<>(); public CompositeCondition() { } public void preMatchHook(RuleEvaluation e) { super.preMatchHook(e); - if (e.getTraceLevel()>=3) { - e.trace(3,"Evaluating '" + this + "'" + " at " + e.currentItem()); + if (e.getTraceLevel() >= 3) { + e.trace(3, "Evaluating '" + this + "'" + " at " + e.currentItem()); e.indentTrace(); } } public void postMatchHook(RuleEvaluation e) { - if (e.getTraceLevel()>=3) { + if (e.getTraceLevel() >= 3) { e.unindentTrace(); } } protected boolean hasOpenChoicepoint(RuleEvaluation evaluation) { - for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) { - Condition subCondition=i.next(); + for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) { + Condition subCondition = i.next(); if (subCondition.hasOpenChoicepoint(evaluation)) return true; } @@ -48,8 +49,8 @@ public abstract class CompositeCondition extends Condition { } /** Sets the condition at the given index */ - public void setCondition(int index,Condition condition) { - conditions.set(index,condition); + public void setCondition(int index, Condition condition) { + conditions.set(index, condition); } /** Returns the number of subconditions */ @@ -74,7 +75,7 @@ public abstract class CompositeCondition extends Condition { * @throws IndexOutOfBoundsException if there is no condition at this index */ public Condition removeCondition(int i) { - Condition condition=conditions.remove(i); + Condition condition = conditions.remove(i); condition.setParent(null); return condition; } @@ -82,20 +83,22 @@ public abstract class CompositeCondition extends Condition { /** Returns an iterator of the immediate children of this condition */ public Iterator<Condition> conditionIterator() { return conditions.iterator(); } + public List<Condition> conditions() { return Collections.unmodifiableList(conditions); } + public void makeReferences(RuleBase rules) { - for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) { - Condition condition=i.next(); + for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) { + Condition condition = i.next(); condition.makeReferences(rules); } } /** Whether this should be output with parentheses, default is parent!=null */ protected boolean useParentheses() { - return getParent()!=null; + return getParent() != null; } protected String toInnerString(String conditionSeparator) { - if (getLabel()!=null) + if (getLabel() != null) return getLabel() + ":(" + conditionsToString(conditionSeparator) + ")"; else if (useParentheses()) return "(" + conditionsToString(conditionSeparator) + ")"; @@ -104,8 +107,8 @@ public abstract class CompositeCondition extends Condition { } protected final String conditionsToString(String conditionSeparator) { - StringBuilder buffer=new StringBuilder(); - for (Iterator<Condition> i=conditionIterator(); i.hasNext(); ) { + StringBuilder buffer = new StringBuilder(); + for (Iterator<Condition> i = conditionIterator(); i.hasNext(); ) { buffer.append(i.next().toString()); if (i.hasNext()) buffer.append(conditionSeparator); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java index 0d3c93619b5..8d6016cb060 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Condition.java @@ -14,20 +14,20 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; public abstract class Condition { /** The parent of this condition, or null if this is not nested */ - private CompositeCondition parent=null; + private CompositeCondition parent = null; /** * The label of this condition, or null if none. * Specified by label:condition * The label is also the default context is no context is speficied explicitly */ - private String label=null; + private String label; /** * The name space refered by this match, or null if the default (query) * Specified by namespace.condition in rules. */ - private String nameSpace=null; + private String nameSpace = null; /** * The name of the context created by this, or null if none @@ -36,9 +36,9 @@ public abstract class Condition { private String contextName; /** Position constraints of the terms matched by this condition */ - private Anchor anchor=Anchor.NONE; + private Anchor anchor = Anchor.NONE; - public static enum Anchor { + public enum Anchor { NONE, START, END, BOTH; public static Anchor create(boolean start,boolean end) { if (start && end) return Anchor.BOTH; @@ -49,32 +49,32 @@ public abstract class Condition { } public Condition() { - this(null,null); + this(null, null); } public Condition(String label) { - this(label,null); + this(label, null); } - public Condition(String label,String context) { - this.label=label; - this.contextName=context; + public Condition(String label, String context) { + this.label = label; + this.contextName = context; } /** * Sets the name whatever is matched by this condition can be refered as, or null - * to make it unreferable + * to make it nonreferable */ - public void setContextName(String contextName) { this.contextName=contextName; } + public void setContextName(String contextName) { this.contextName = contextName; } /** - * Returns the name whatever is matched by this condition can be refered as, or null + * Returns the name whatever is matched by this condition can be referred as, or null * if it is unreferable */ public String getContextName() { return contextName; } /** Returns whether this is referable, returns context!=null by default */ - protected boolean isReferable() { return contextName!=null; } + protected boolean isReferable() { return contextName != null; } /** Sets the label of this. Set to null to use the default */ public String getLabel() { return label; } @@ -95,15 +95,15 @@ public abstract class Condition { void setParent(CompositeCondition parent) { this.parent=parent; } /** Sets a positional constraint on this condition */ - public void setAnchor(Anchor anchor) { this.anchor=anchor; } + public void setAnchor(Anchor anchor) { this.anchor = anchor; } /** Returns the positional constraint on this anchor. This is never null */ public Anchor getAnchor() { return anchor; } /** - * <p>Returns whether this condition matches the given evaluation + * Returns whether this condition matches the given evaluation * at the <i>current</i> location of the evaluation. Calls the doesMatch - * method of each condition subtype.</p> + * method of each condition subtype. */ public final boolean matches(RuleEvaluation e) { // TODO: With this algoritm, each choice point will move to the next choice on each reevaluation @@ -138,32 +138,32 @@ public abstract class Condition { /** Check start anchor. Trace level 4 if no match */ protected boolean matchesStartAnchor(RuleEvaluation e) { - if (anchor!=Anchor.START && anchor!=Anchor.BOTH) return true; - if (e.getPosition()==0) return true; - if (e.getTraceLevel()>=4) - e.trace(4,this + " must be at the start, which " + e.currentItem() + " isn't"); + if (anchor != Anchor.START && anchor != Anchor.BOTH) return true; + if (e.getPosition() == 0) return true; + if (e.getTraceLevel() >= 4) + e.trace(4, this + " must be at the start, which " + e.currentItem() + " isn't"); return false; } /** Check start anchor. Trace level 4 if no match */ protected boolean matchesEndAnchor(RuleEvaluation e) { - if (anchor!=Anchor.END && anchor!=Anchor.BOTH) return true; - if (e.getPosition()>=e.items().size()) return true; - if (e.getTraceLevel()>=4) - e.trace(4,this + " must be at the end, which " + e.currentItem() + " isn't"); + if (anchor != Anchor.END && anchor != Anchor.BOTH) return true; + if (e.getPosition() >= e.items().size()) return true; + if (e.getTraceLevel() >= 4) + e.trace(4, this + " must be at the end, which " + e.currentItem() + " isn't"); return false; } - protected void traceResult(boolean matches,RuleEvaluation e) { - if (matches && e.getTraceLevel()>=3) - e.trace(3,"Matched '" + this + "'" + getMatchInfoString(e) + " at " + e.previousItem()); - if (!matches && e.getTraceLevel()>=4) - e.trace(4,"Did not match '" + this + "' at " + e.currentItem()); + protected void traceResult(boolean matches, RuleEvaluation e) { + if (matches && e.getTraceLevel() >= 3) + e.trace(3, "Matched '" + this + "'" + getMatchInfoString(e) + " at " + e.previousItem()); + if (!matches && e.getTraceLevel() >= 4) + e.trace(4, "Did not match '" + this + "' at " + e.currentItem()); } protected String getMatchInfoString(RuleEvaluation e) { - String matchInfo=getMatchInfo(e); - if (matchInfo==null) return ""; + String matchInfo = getMatchInfo(e); + if (matchInfo == null) return ""; return " as '" + matchInfo + "'"; } @@ -212,28 +212,28 @@ public abstract class Condition { public void makeReferences(RuleBase rules) { } protected String getLabelString() { - if (label==null) return ""; + if (label == null) return ""; return label + ":"; } /** Whether the label matches the current item, true if there is no current item */ protected boolean labelMatches(RuleEvaluation e) { - FlattenedItem flattenedItem=e.currentItem(); - if (flattenedItem==null) return true; - TermItem item=flattenedItem.getItem(); - if (item==null) return true; - return labelMatches(item,e); - } - - protected boolean labelMatches(TermItem evaluationTerm,RuleEvaluation e) { - String indexName=evaluationTerm.getIndexName(); - String label=getLabel(); - if (label==null) - label=e.getCurrentLabel(); - if ("".equals(indexName) && label==null) return true; + FlattenedItem flattenedItem = e.currentItem(); + if (flattenedItem == null) return true; + TermItem item = flattenedItem.getItem(); + if (item == null) return true; + return labelMatches(item, e); + } + + protected boolean labelMatches(TermItem evaluationTerm, RuleEvaluation e) { + String indexName = evaluationTerm.getIndexName(); + String label = getLabel(); + if (label == null) + label = e.getCurrentLabel(); + if ("".equals(indexName) && label == null) return true; if (indexName.equals(label)) return true; - if (e.getTraceLevel()>=4) - e.trace(4,"'" + this + "' does not match, label of " + e.currentItem() + " was required to be " + label); + if (e.getTraceLevel() >= 4) + e.trace(4, "'" + this + "' does not match, label of " + e.currentItem() + " was required to be " + label); return false; } @@ -242,13 +242,14 @@ public abstract class Condition { protected boolean isDefaultContextName() { return false; } + @Override public String toString() { - String contextString=""; - String nameSpaceString=""; - if (contextName!=null && !isDefaultContextName()) - contextString=contextName + "/"; - if (getNameSpace()!=null) - nameSpaceString=getNameSpace() + "."; + String contextString = ""; + String nameSpaceString = ""; + if (contextName != null && !isDefaultContextName()) + contextString = contextName + "/"; + if (getNameSpace() != null) + nameSpaceString = getNameSpace() + "."; return contextString + nameSpaceString + toInnerString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java index bfddb55566c..e468526fbe0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ConditionReference.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; -import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.querytransform.PhraseMatcher; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleBaseException; @@ -11,8 +10,6 @@ import com.yahoo.prelude.semantics.engine.FlattenedItem; import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; -import java.util.Map; - /** * A reference to a named condition * @@ -24,7 +21,7 @@ public class ConditionReference extends Condition { private String conditionName; /** - * The actual condition references by this, or null if not initialized or not found, + * The actual condition referenced by this, or null if not initialized or not found, * or if this is really an automata reference */ private NamedCondition namedCondition; @@ -33,7 +30,7 @@ public class ConditionReference extends Condition { * True if this condition should be looked up in the automata * annotations of the item instead of by reference to another item */ - private boolean automataLookup=false; + private boolean automataLookup = false; public ConditionReference(String conditionName) { this(null,conditionName); @@ -42,37 +39,36 @@ public class ConditionReference extends Condition { public ConditionReference(String label,String conditionName) { super(label); Validator.ensureNotNull("Name of referenced condition",conditionName); - this.conditionName=conditionName; + this.conditionName = conditionName; setContextName(conditionName); } /** Returns the name of the referenced rule, never null */ public String getConditionName() { return conditionName; } - public void setConditionName(String name) { this.conditionName=name; } + public void setConditionName(String name) { this.conditionName = name; } public boolean doesMatch(RuleEvaluation e) { if (automataLookup) return automataMatch(e); - if (namedCondition==null) - throw new EvaluationException("Condition reference '" + conditionName + - "' not found or not initialized"); + if (namedCondition == null) + throw new EvaluationException("Condition reference '" + conditionName + "' not found or not initialized"); return namedCondition.matches(e); } private boolean automataMatch(RuleEvaluation e) { - FlattenedItem current=e.currentItem(); - if (current==null) return false; + FlattenedItem current = e.currentItem(); + if (current == null) return false; - Object annotation=current.getItem().getAnnotation(conditionName); - if (annotation==null) return false; + Object annotation = current.getItem().getAnnotation(conditionName); + if (annotation == null) return false; if (! (annotation instanceof PhraseMatcher.Phrase)) return false; - PhraseMatcher.Phrase phrase=(PhraseMatcher.Phrase)annotation; + PhraseMatcher.Phrase phrase = (PhraseMatcher.Phrase)annotation; - Choicepoint choicePoint=e.getChoicepoint(this,true); - boolean matches=automataMatchPhrase(phrase,e); + Choicepoint choicePoint = e.getChoicepoint(this,true); + boolean matches = automataMatchPhrase(phrase,e); if (!matches && e.isInNegation()) { // TODO: Temporary hack! Works for single items only e.addMatch(current,null); @@ -84,40 +80,41 @@ public class ConditionReference extends Condition { return matches; } - private boolean automataMatchPhrase(PhraseMatcher.Phrase phrase,RuleEvaluation e) { - for (PhraseMatcher.Phrase.MatchIterator i=phrase.itemIterator(); i.hasNext(); ) { + private boolean automataMatchPhrase(PhraseMatcher.Phrase phrase, RuleEvaluation e) { + for (PhraseMatcher.Phrase.MatchIterator i = phrase.itemIterator(); i.hasNext(); ) { i.next(); - FlattenedItem current=e.currentItem(); - if (current==null) return false; + FlattenedItem current = e.currentItem(); + if (current == null) return false; if (!labelMatches(e.currentItem().getItem(),e)) return false; if (!e.isInNegation()) - e.addMatch(current,i.getReplace()); + e.addMatch(current, i.getReplace()); e.next(); } - if (phrase.getLength()>phrase.getBackedLength()) return false; // The underlying composite item has changed + if (phrase.getLength() > phrase.getBackedLength()) return false; // The underlying composite item has changed return true; } public void makeReferences(RuleBase ruleBase) { - namedCondition=ruleBase.getCondition(conditionName); - if (namedCondition==null) { // Then this may reference some automata value, if we have an automata + namedCondition = ruleBase.getCondition(conditionName); + if (namedCondition == null) { // Then this may reference some automata value, if we have an automata if (ruleBase.usesAutomata()) - automataLookup=true; + automataLookup = true; else throw new RuleBaseException("Referenced condition '" + conditionName + - "' does not exist in " + ruleBase); + "' does not exist in " + ruleBase); } } protected boolean hasOpenChoicepoint(RuleEvaluation e) { - if (namedCondition==null) return false; + if (namedCondition == null) return false; return namedCondition.getCondition().hasOpenChoicepoint(e); } protected boolean isDefaultContextName() { - return getContextName()==null || getContextName().equals(conditionName); + return getContextName() == null || getContextName().equals(conditionName); } + @Override protected String toInnerString() { return "[" + conditionName + "]"; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java index 0d5ce78baf0..af7aab23b85 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralPhraseProduction.java @@ -44,7 +44,7 @@ public class LiteralPhraseProduction extends TermProduction { /** Returns a read only view of the terms produced by this, never null */ public List<String> getTerms() { return Collections.unmodifiableList(terms); } - public void produce(RuleEvaluation e,int offset) { + public void produce(RuleEvaluation e, int offset) { PhraseItem newPhrase = new PhraseItem(); newPhrase.setIndexName(getLabel()); for (String term : terms) @@ -52,13 +52,13 @@ public class LiteralPhraseProduction extends TermProduction { if (replacing) { Match matched = e.getNonreferencedMatch(0); - insertMatch(e, matched, newPhrase, offset); + insertMatch(e, matched, List.of(newPhrase), offset); } else { newPhrase.setWeight(getWeight()); if (e.getTraceLevel() >= 6) e.trace(6, "Adding '" + newPhrase + "'"); - e.addItem(newPhrase, getTermType()); + e.addItems(List.of(newPhrase), getTermType()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java index ed21c9643c3..66507719a11 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralTermProduction.java @@ -7,6 +7,8 @@ import com.yahoo.prelude.semantics.engine.Match; import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; +import java.util.List; + /** * A literal term produced by a production rule * @@ -63,13 +65,13 @@ public class LiteralTermProduction extends TermProduction { if (replacing) { Match matched = e.getNonreferencedMatch(0); newItem.setWeight(matched.getItem().getWeight()); - insertMatch(e, matched, newItem, offset); + insertMatch(e, matched, List.of(newItem), offset); } else { newItem.setWeight(getWeight()); if (e.getTraceLevel() >= 6) e.trace(6, "Adding '" + newItem + "'"); - e.addItem(newItem, getTermType()); + e.addItems(List.of(newItem), getTermType()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java index a267d274d5a..cbb0b4ab10a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java @@ -33,7 +33,7 @@ public class NamedCondition { e.indentTrace(); } - boolean matches=condition.matches(e); + boolean matches = condition.matches(e); if (e.getTraceLevel() >= 3) { e.unindentTrace(); @@ -50,6 +50,7 @@ public class NamedCondition { * This string representation can always be reparsed to produce an * identical rule to this one. */ + @Override public String toString() { return "[" + conditionName + "] :- " + condition.toString(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java index a971020ea90..debc3150959 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/Production.java @@ -13,20 +13,20 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; public abstract class Production { /** True to add, false to replace, default true */ - protected boolean replacing=true; + protected boolean replacing = true; /** The (0-base) position of this term in the productions of this rule */ - private int position=0; + private int position = 0; /** The weight (strength) of this production as a percentage (default is 100) */ - private int weight=100; + private int weight = 100; /** Creates a produced template term with no label and the default type */ public Production() { } /** True to replace, false to add, if this production can do both. Default true. */ - public void setReplacing(boolean replacing) { this.replacing=replacing; } + public void setReplacing(boolean replacing) { this.replacing = replacing; } public int getPosition() { return position; } @@ -45,7 +45,7 @@ public abstract class Production { * @param offset the offset position at which to produce this. Offsets are used to produce multiple items * at one position, inserted in the right order. */ - public abstract void produce(RuleEvaluation e,int offset); + public abstract void produce(RuleEvaluation e, int offset); /** * Called to add the references into the condition of this rule made by this production @@ -55,6 +55,7 @@ public abstract class Production { void addMatchReferences(Set<String> matchReferences) { } /** All instances of this produces a parseable string output */ + @Override public final String toString() { return toInnerString() + (getWeight()!=100 ? ("!" + getWeight()) : ""); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java index 46f855ca95d..5d20075cd97 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionList.java @@ -54,6 +54,7 @@ public class ProductionList { } } + @Override public String toString() { StringBuilder buffer = new StringBuilder(); for (Iterator<Production> i = productions.iterator(); i.hasNext(); ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java index 20ebc41e57f..b320d10b43d 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ProductionRule.java @@ -19,18 +19,18 @@ public abstract class ProductionRule { private Condition condition; /** What is produced when this rule is true */ - private ProductionList production=new ProductionList(); + private ProductionList production = new ProductionList(); /** The set of match name Strings which the production part of this rule references */ - private Set<String> matchReferences=new java.util.LinkedHashSet<>(); + private final Set<String> matchReferences = new java.util.LinkedHashSet<>(); /** Sets what must be true for this rule to be true */ - public void setCondition(Condition condition) { this.condition=condition; } + public void setCondition(Condition condition) { this.condition = condition; } public Condition getCondition() { return condition; } /** Sets what is produced when this rule is true */ - public void setProduction(ProductionList production) { this.production=production; } + public void setProduction(ProductionList production) { this.production = production; } public ProductionList getProduction() { return production; } @@ -64,6 +64,7 @@ public abstract class ProductionRule { * This string representation can always be reparsed to produce an * identical rule to this one. */ + @Override public String toString() { return condition.toString() + " " + getSymbol() + " " + production.toString(); } @@ -80,21 +81,27 @@ public abstract class ProductionRule { * This default implementation returns false; */ public boolean isLoop() { - // TODO: There are many more possible loops, we should probably detect - // a few more obvious ones + // TODO: There are many more possible loops, we should probably detect a few more obvious ones if (conditionIsEllipsAndOtherNameSpacesOnly(getCondition())) return true; + if (producesItself()) return true; return false; } private boolean conditionIsEllipsAndOtherNameSpacesOnly(Condition condition) { if (condition instanceof EllipsisCondition) return true; if (! (condition instanceof CompositeCondition)) return false; - for (Iterator<Condition> i=((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { - Condition child= i.next(); - if (child.getNameSpace()==null && conditionIsEllipsAndOtherNameSpacesOnly(child)) + for (Iterator<Condition> i = ((CompositeCondition)condition).conditionIterator(); i.hasNext(); ) { + Condition child = i.next(); + if (child.getNameSpace() == null && conditionIsEllipsAndOtherNameSpacesOnly(child)) return true; } return false; } + private boolean producesItself() { + return production.productionList() + .stream() + .anyMatch(p -> (p instanceof ReferenceTermProduction) && ((ReferenceTermProduction)p).producesAll()); + } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java index af7abf325e7..00562d3953e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java @@ -1,15 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; import java.util.Set; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.TermType; import com.yahoo.prelude.semantics.engine.EvaluationException; import com.yahoo.prelude.semantics.engine.Match; import com.yahoo.prelude.semantics.engine.ReferencedMatches; import com.yahoo.prelude.semantics.engine.RuleEvaluation; -import com.yahoo.protect.Validator; /** * A term produced by a production rule which takes its actual term value @@ -19,90 +23,114 @@ import com.yahoo.protect.Validator; */ public class ReferenceTermProduction extends TermProduction { - private String reference; + private final String reference; + private final boolean produceAll; /** * Creates a new produced reference term * * @param reference the label of the condition this should take it's value from */ - public ReferenceTermProduction(String reference) { + public ReferenceTermProduction(String reference, boolean produceAll) { super(); - setReference(reference); + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** * Creates a new produced reference term * - * @param reference the label of the condition this should take it's value from + * @param reference the label of the condition this should take its value from * @param termType the type of the term to produce */ - public ReferenceTermProduction(String reference, TermType termType) { + public ReferenceTermProduction(String reference, TermType termType, boolean produceAll) { super(termType); - setReference(reference); + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** * Creates a new produced reference term * * @param label the label of the produced term - * @param reference the label of the condition this should take it's value from + * @param reference the label of the condition this should take its value from */ - public ReferenceTermProduction(String label, String reference) { + public ReferenceTermProduction(String label, String reference, boolean produceAll) { super(label); - setReference(reference); + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** * Creates a new produced reference term * * @param label the label of the produced term - * @param reference the label of the condition this should take it's value from + * @param reference the label of the condition this should take its value from * @param termType the type of term to produce */ - public ReferenceTermProduction(String label, String reference, TermType termType) { + public ReferenceTermProduction(String label, String reference, TermType termType, boolean produceAll) { super(label, termType); - setReference(reference); - } - - /** The label of the condition this should take its value from, never null */ - public void setReference(String reference) { - Validator.ensureNotNull("reference name of a produced reference term",reference); - this.reference = reference; + this.reference = Objects.requireNonNull(reference, "Reference cannot be null"); + this.produceAll = produceAll; } /** Returns the label of the condition this should take its value from, never null */ public String getReference() { return reference; } + public boolean producesAll() { return produceAll; } + + @Override void addMatchReferences(Set<String> matchReferences) { matchReferences.add(reference); } - public void produce(RuleEvaluation e, int offset) { + public void produce(RuleEvaluation e, int ignored) { ReferencedMatches referencedMatches = e.getReferencedMatches(reference); if (referencedMatches == null) throw new EvaluationException("Referred match '" + reference + "' not found"); if (replacing) { - replaceMatches(e, referencedMatches); + e.removeMatches(referencedMatches); + } + + var match = referencedMatches.matchIterator().next(); + if (produceAll) { + // produce all terms in the condition + NamedCondition namedCondition = e.getEvaluation().ruleBase().getCondition(referencedMatches.getContextName()); + ChoiceCondition choices = (ChoiceCondition)namedCondition.getCondition(); + List<Item> items = new ArrayList<>(); + for (Iterator<Condition> i = choices.conditionIterator(); i.hasNext();) { + Condition condition = i.next(); + if (condition instanceof TermCondition) { + items.add(match.toItem(getLabel(), ((TermCondition)condition).term())); + } + else if (condition instanceof SequenceCondition) { + PhraseItem phrase = new PhraseItem(getLabel()); + for (var term : ((SequenceCondition)condition).conditions()) + phrase.addItem(match.toItem(getLabel(), ((TermCondition)term).term())); + items.add(phrase); + } + else { + // Until we validate this at construction time + throw new EvaluationException("Could not produce all terms in " + namedCondition + " as it is " + + "not a term or sequence condition"); + } + } + produce(e, match, items, 0); } else { - addMatches(e, referencedMatches); + // produce just the matching term + produce(e, match, List.of(referencedMatches.toItem(getLabel())), 0); } } - public void replaceMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { - Item referencedItem = referencedMatches.toItem(getLabel()); - if (referencedItem == null) return; - e.removeMatches(referencedMatches); - insertMatch(e, referencedMatches.matchIterator().next(), referencedItem, 0); - } - - private void addMatches(RuleEvaluation e, ReferencedMatches referencedMatches) { - Item referencedItem = referencedMatches.toItem(getLabel()); - if (referencedItem == null) return; - e.addItem(referencedItem, getTermType()); + private void produce(RuleEvaluation e, Match match, List<Item> items, int offset) { + if (replacing) + insertMatch(e, match, items, offset); + else + e.addItems(items, getTermType()); } + @Override public String toInnerTermString() { return getLabelString() + "[" + reference + "]"; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java index a2bbf72a53b..eaff66a0bb0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.rule; -import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.semantics.engine.NameSpace; import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.semantics.engine.RuleEvaluation; @@ -49,6 +48,9 @@ public class TermCondition extends Condition { } } + public String term() { return term; } + + @Override public String toInnerString() { return getLabelString() + term; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java index 29e4982ac17..41d15bc9262 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java @@ -7,6 +7,8 @@ import com.yahoo.prelude.semantics.engine.Match; import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; +import java.util.List; + /** * A new term produced by a production rule * @@ -59,9 +61,9 @@ public abstract class TermProduction extends Production { * Inserts newItem at the position of this match * TODO: Move to ruleevaluation */ - protected void insertMatch(RuleEvaluation e, Match matched, Item newItem, int offset) { + protected void insertMatch(RuleEvaluation e, Match matched, List<Item> newItems, int offset) { if (getWeight() != 100) - newItem.setWeight(getWeight()); + newItems.forEach(item -> item.setWeight(getWeight())); int insertPosition = matched.getPosition() + offset; // This check is necessary (?) because earlier items may have been removed @@ -71,9 +73,9 @@ public abstract class TermProduction extends Production { insertPosition = matched.getParent().getItemCount(); } - e.insertItem(newItem, matched.getParent(), insertPosition,getTermType()); + e.insertItems(newItems, matched.getParent(), insertPosition, getTermType()); if (e.getTraceLevel() >= 6) - e.trace(6, "Inserted item '" + newItem + "' at position " + insertPosition + " producing " + + e.trace(6, "Inserted items '" + newItems + "' at position " + insertPosition + " producing " + e.getEvaluation().getQuery().getModel().getQueryTree()); } diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 1d1c446f6e1..6fae5c97cd2 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -72,6 +72,7 @@ public class StatisticsSearcher extends Searcher { private static final String RELEVANCE_AT_1_METRIC = "relevance.at_1"; private static final String RELEVANCE_AT_3_METRIC = "relevance.at_3"; private static final String RELEVANCE_AT_10_METRIC = "relevance.at_10"; + private static final String QUERY_ITEM_COUNT = "query_item_count"; @SuppressWarnings("unused") // all the work is done by the callback private final PeakQpsReporter peakQpsReporter; @@ -264,6 +265,8 @@ public class StatisticsSearcher extends Searcher { addRelevanceMetrics(query, execution, result); + addItemCountMetric(query, metricContext); + return result; } @@ -396,6 +399,10 @@ public class StatisticsSearcher extends Searcher { } } + private void addItemCountMetric(Query query, Metric.Context context) { + metric.set(QUERY_ITEM_COUNT, query.getModel().getQueryTree().treeSize(), context); + } + /** * Returns the relative start time from request was received by jdisc */ diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 623c38fa9f0..1c38bb8f876 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -235,18 +235,18 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private static final Map<String, CompoundName> propertyAliases; static { Map<String,CompoundName> propertyAliasesBuilder = new HashMap<>(); - addAliases(Query.getArgumentType(), propertyAliasesBuilder); - addAliases(Ranking.getArgumentType(), propertyAliasesBuilder); - addAliases(Model.getArgumentType(), propertyAliasesBuilder); - addAliases(Presentation.getArgumentType(), propertyAliasesBuilder); - addAliases(Select.getArgumentType(), propertyAliasesBuilder); + addAliases(Query.getArgumentType(), CompoundName.empty, propertyAliasesBuilder); propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder); } - private static void addAliases(QueryProfileType arguments, Map<String, CompoundName> aliases) { - CompoundName prefix = getPrefix(arguments); + private static void addAliases(QueryProfileType arguments, CompoundName prefix, Map<String, CompoundName> aliases) { for (FieldDescription field : arguments.fields().values()) { for (String alias : field.getAliases()) aliases.put(alias, prefix.append(field.getName())); + if (field.getType() instanceof QueryProfileFieldType) { + var type = ((QueryProfileFieldType) field.getType()).getQueryProfileType(); + if (type != null) + addAliases(type, prefix.append(type.getComponentIdAsCompoundName()), aliases); + } } } @@ -1048,6 +1048,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { return new SessionId(requestId, getRanking().getProfile()); } + @Deprecated // TODO: Remove on Vespa 8 public boolean hasEncodableProperties() { if ( ! ranking.getProperties().isEmpty()) return true; if ( ! ranking.getFeatures().isEmpty()) return true; @@ -1064,39 +1065,29 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * @param buffer the buffer to encode to * @param encodeQueryData true to encode all properties, false to only include session information, not actual query data * @return the encoded length + * @deprecated do not use */ + @Deprecated // TODO: Remove on Vespa 8 public int encodeAsProperties(ByteBuffer buffer, boolean encodeQueryData) { // Make sure we don't encode anything here if we have turned the property feature off // Due to sendQuery we sometimes end up turning this feature on and then encoding a 0 int as the number of // property maps - that's ok (probably we should simplify by just always turning the feature on) if (! hasEncodableProperties()) return 0; - int start = buffer.position(); - int mapCountPosition = buffer.position(); buffer.putInt(0); // map count will go here - int mapCount = 0; - - // TODO: Push down mapCount += ranking.getProperties().encode(buffer, encodeQueryData); if (encodeQueryData) { mapCount += ranking.getFeatures().encode(buffer); - - // TODO: Push down if (presentation.getHighlight() != null) { mapCount += MapEncoder.encodeMultiMap(Highlight.HIGHLIGHTTERMS, presentation.getHighlight().getHighlightTerms(), buffer); } - - // TODO: Push down mapCount += MapEncoder.encodeMap("model", createModelMap(), buffer); } mapCount += MapEncoder.encodeSingleValue(DocumentDatabase.MATCH_PROPERTY, DocumentDatabase.SEARCH_DOC_TYPE_KEY, model.getDocumentDb(), buffer); - mapCount += MapEncoder.encodeMap("caches", createCacheSettingMap(), buffer); - buffer.putInt(mapCountPosition, mapCount); - return buffer.position() - start; } @@ -1126,7 +1117,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** * Prepares this for binary serialization. - * <p> + * * This must be invoked after all changes have been made to this query before it is passed * on to a receiving backend. Calling it is somewhat expensive, so it should only happen once. * If a prepared query is cloned, it stays prepared. diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 016e22839c1..360dcd38d3b 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -47,7 +47,7 @@ public class ClusterMonitor<T> { public ClusterMonitor(NodeManager<T> manager, boolean startPingThread) { nodeManager = manager; - monitorThread = new MonitorThread("search.clustermonitor"); + monitorThread = new MonitorThread("search.clustermonitor." + manager.name()); if (startPingThread) { monitorThread.start(); } @@ -81,7 +81,9 @@ public class ClusterMonitor<T> { /** * Returns the monitor of the given node, or null if this node has not been added + * @deprecated Will be removed in Vespa 8. */ + @Deprecated(forRemoval = true, since = "7.537") public BaseNodeMonitor<T> getNodeMonitor(T node) { return nodeMonitors.get(node); } @@ -147,6 +149,7 @@ public class ClusterMonitor<T> { private class MonitorThread extends Thread { MonitorThread(String name) { super(name); + setDaemon(true); } public void run() { diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 51ae2ee432d..097d714b47b 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -51,7 +51,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod * @param internal whether or not this cluster is internal (part of the same installation) */ public ClusterSearcher(ComponentId id, List<T> connections, boolean internal) { - this(id, connections, new Hasher<T>(), internal); + this(id, connections, new Hasher<>(), internal); } public ClusterSearcher(ComponentId id, List<T> connections, Hasher<T> hasher, boolean internal) { @@ -68,6 +68,9 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod } } + @Override + public String name() { return getIdString(); } + /** Pinging a node, called from ClusterMonitor */ @Override public final void ping(ClusterMonitor<T> clusterMonitor, T p, Executor executor) { @@ -112,7 +115,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (k == null) { b.append("null\n"); } else { - b.append(k.toString()).append('\n'); + b.append(k).append('\n'); } } traceAsString = b.toString(); @@ -303,7 +306,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod private class Pinger implements Callable<Pong> { - private T connection; + private final T connection; public Pinger(T connection) { this.connection = connection; diff --git a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java index 1a74194d694..db583a65606 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/NodeManager.java @@ -11,6 +11,9 @@ import java.util.concurrent.Executor; */ public interface NodeManager<T> { + /** Name to identify Nodemanager */ + default String name() { return ""; } + /** Called when a failed node is working (ready for production) again */ void working(T node); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 3cf2903b88f..e09b1d51733 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -46,17 +46,12 @@ import java.util.stream.Collectors; public class Dispatcher extends AbstractComponent { public static final String DISPATCH = "dispatch"; - private static final String INTERNAL = "internal"; - private static final String PROTOBUF = "protobuf"; private static final String TOP_K_PROBABILITY = "topKProbability"; private static final String INTERNAL_METRIC = "dispatch_internal"; private static final int MAX_GROUP_SELECTION_ATTEMPTS = 3; - /** If enabled, search queries will use protobuf rpc */ - public static final CompoundName dispatchProtobuf = CompoundName.fromComponents(DISPATCH, PROTOBUF); - /** If set will control computation of how many hits will be fetched from each partition.*/ public static final CompoundName topKProbability = CompoundName.fromComponents(DISPATCH, TOP_K_PROBABILITY); @@ -79,8 +74,6 @@ public class Dispatcher extends AbstractComponent { argumentType = new QueryProfileType(DISPATCH); argumentType.setStrict(true); argumentType.setBuiltin(true); - argumentType.addField(new FieldDescription(INTERNAL, FieldType.booleanType)); - argumentType.addField(new FieldDescription(PROTOBUF, FieldType.booleanType)); argumentType.addField(new FieldDescription(TOP_K_PROBABILITY, FieldType.doubleType)); argumentType.freeze(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java index 03b0f092abb..bd0415fa449 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LeanHit.java @@ -11,17 +11,17 @@ import java.util.Arrays; */ public class LeanHit implements Comparable<LeanHit> { - private final byte [] gid; + private final byte[] gid; private final double relevance; - private final byte [] sortData; + private final byte[] sortData; private final int partId; private final int distributionKey; private FeatureData matchFeatures; - public LeanHit(byte [] gid, int partId, int distributionKey, double relevance) { + public LeanHit(byte[] gid, int partId, int distributionKey, double relevance) { this(gid, partId, distributionKey, relevance, null); } - public LeanHit(byte [] gid, int partId, int distributionKey, double relevance, byte [] sortData) { + public LeanHit(byte[] gid, int partId, int distributionKey, double relevance, byte[] sortData) { this.gid = gid; this.relevance = Double.isNaN(relevance) ? Double.NEGATIVE_INFINITY : relevance; this.sortData = sortData; @@ -31,8 +31,8 @@ public class LeanHit implements Comparable<LeanHit> { } public double getRelevance() { return relevance; } - public byte [] getGid() { return gid; } - public byte [] getSortData() { return sortData; } + public byte[] getGid() { return gid; } + public byte[] getSortData() { return sortData; } public boolean hasSortData() { return sortData != null; } public int getPartId() { return partId; } public int getDistributionKey() { return distributionKey; } @@ -51,7 +51,7 @@ public class LeanHit implements Comparable<LeanHit> { return (res != 0) ? res : compareData(gid, o.gid); } - private static int compareData(byte [] left, byte [] right) { + private static int compareData(byte[] left, byte[] right) { int i = Arrays.mismatch(left, right); if (i < 0) { return 0; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index fe74180cad3..7a5ef94069d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -65,6 +65,11 @@ public class RpcProtobufFillInvoker extends FillInvoker { @Override protected void sendFillRequest(Result result, String summaryClass) { + if (! documentDb.getDocsumDefinitionSet().hasDocsum(summaryClass)) { + // TODO Vespa 8: + // throw new IllegalArgumentException("invalid summary="+summaryClass); + log.fine("invalid presentation.summary="+summaryClass); + } ListMap<Integer, FastHit> hitsByNode = hitsByNode(result); result.getQuery().trace(false, 5, "Sending ", hitsByNode.size(), " summary fetch requests with jrt/protobuf"); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index f90abba330a..36d7e7a85a9 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -84,6 +84,9 @@ public class SearchCluster implements NodeManager<Node> { this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), nodesByHost, groups); } + @Override + public String name() { return clusterId; } + public void addMonitoring(ClusterMonitor<Node> clusterMonitor) { for (var group : orderedGroups()) { for (var node : group.nodes()) diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 8925c647ad2..b65953935f0 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.handler; -import ai.vespa.cloud.Environment; -import ai.vespa.cloud.Zone; import com.google.inject.Inject; import com.yahoo.collections.Tuple2; import com.yahoo.component.ComponentSpecification; @@ -75,6 +73,7 @@ import java.util.logging.Logger; * @author Steinar Knutsen * @author bratseth */ +@SuppressWarnings("deprecation") // super class is deprecated public class SearchHandler extends LoggingRequestHandler { private static final Logger log = Logger.getLogger(SearchHandler.class.getName()); diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index db6b07e0b0d..31641b5c2f0 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -8,7 +8,9 @@ import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.IndexedItem; import com.yahoo.search.Query; import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.ranking.MatchPhase; import com.yahoo.search.rendering.RendererRegistry; import java.util.ArrayList; @@ -30,19 +32,26 @@ public class Presentation implements Cloneable { public static final String TIMING = "timing"; public static final String SUMMARY = "summary"; public static final String SUMMARY_FIELDS = "summaryFields"; + public static final String TENSORS = "tensors"; /** The (short) name of the parameter holding the name of the return format to use */ public static final String FORMAT = "format"; static { - argumentType=new QueryProfileType(PRESENTATION); + argumentType = new QueryProfileType(PRESENTATION); argumentType.setStrict(true); argumentType.setBuiltin(true); argumentType.addField(new FieldDescription(BOLDING, "boolean", "bolding")); argumentType.addField(new FieldDescription(TIMING, "boolean", "timing")); argumentType.addField(new FieldDescription(SUMMARY, "string", "summary")); - argumentType.addField(new FieldDescription(FORMAT, "string", "format template")); argumentType.addField(new FieldDescription(SUMMARY_FIELDS, "string", "summaryFields")); + QueryProfileType formatArgumentType = new QueryProfileType(FORMAT); + formatArgumentType.setBuiltin(true); + formatArgumentType.setStrict(true); + formatArgumentType.addField(new FieldDescription("", "string", "format template")); + formatArgumentType.addField(new FieldDescription(TENSORS, "string", "format.tensors")); + formatArgumentType.freeze(); + argumentType.addField(new FieldDescription(FORMAT, new QueryProfileFieldType(formatArgumentType), "format")); argumentType.freeze(); } public static QueryProfileType getArgumentType() { return argumentType; } @@ -53,7 +62,7 @@ public class Presentation implements Cloneable { /** The terms to highlight in the result (only used by BoldingSearcher, may be removed later). */ private List<IndexedItem> boldingData = null; - /** Whether or not to do highlighting */ + /** Whether to do highlighting */ private boolean bolding = true; /** The summary class to be shown */ @@ -65,6 +74,9 @@ public class Presentation implements Cloneable { /** Whether optional timing data should be rendered */ private boolean timing = false; + /** Whether to renders tensors in short form */ + private boolean tensorShortForm = false; + /** Set of explicitly requested summary fields, instead of summary classes */ private Set<String> summaryFields = LazySet.newHashSet(); @@ -98,14 +110,10 @@ public class Presentation implements Cloneable { this.format = (format != null) ? format : RendererRegistry.defaultRendererId.toSpecification(); } - /** - * Get the name of the format desired for result rendering. - */ + /** Get the name of the format desired for result rendering. */ public String getFormat() { return format.getName(); } - /** - * Set the desired format for result rendering. If null, use the default renderer. - */ + /** Set the desired format for result rendering. If null, use the default renderer. */ public void setFormat(String format) { setRenderer(ComponentSpecification.fromString(format)); } @@ -132,21 +140,7 @@ public class Presentation implements Cloneable { } } - @Override - public boolean equals(Object o) { - if ( ! (o instanceof Presentation)) return false; - Presentation p = (Presentation) o; - return QueryHelper.equals(bolding, p.bolding) && QueryHelper.equals(summary, p.summary); - } - - @Override - public int hashCode() { - return QueryHelper.combineHash(bolding, summary); - } - - /** - * @return whether to add optional timing data to the rendered result - */ + /** Returns whether to add optional timing data to the rendered result. */ public boolean getTiming() { return timing; } @@ -166,20 +160,13 @@ public class Presentation implements Cloneable { return summaryFields; } - /** Prepares this for binary serialization. For internal use - see {@link Query#prepare} */ - public void prepare() { - if (highlight != null) - highlight.prepare(); - } - /** * Parse the given string as a comma delimited set of field names and * overwrite the set of summary fields. Whitespace will be trimmed. If you * want to add or remove fields programmatically, use * {@link #getSummaryFields()} and modify the returned set. * - * @param asString - * the summary fields requested, e.g. "price,author,title" + * @param asString the summary fields requested, e.g. "price,author,title" */ public void setSummaryFields(String asString) { summaryFields.clear(); @@ -189,5 +176,53 @@ public class Presentation implements Cloneable { } + /** + * Returns whether tensors should use short form in JSON and textual representations, see + * <a href="https://docs.vespa.ai/en/reference/document-json-format.html#tensor">https://docs.vespa.ai/en/reference/document-json-format.html#tensor</a> + * and <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form</a>. + * Default is false. + */ + public boolean getTensorShortForm() { return tensorShortForm; } + + /** + * Sets whether tensors should use short form in JSON and textual representations from a string. + * + * @param value a string which must be either 'short' or 'long' + * @throws IllegalArgumentException if any other value is passed + */ + public void setTensorShortForm(String value) { + tensorShortForm = toTensorShortForm(value); + } + + private boolean toTensorShortForm(String value) { + switch (value) { + case "short": return true; + case "long": return false; + default: throw new IllegalArgumentException("Value must be 'long' or 'short', not '" + value + "'"); + } + } + + public void setTensorShortForm(boolean tensorShortForm) { + this.tensorShortForm = tensorShortForm; + } + + /** Prepares this for binary serialization. For internal use - see {@link Query#prepare} */ + public void prepare() { + if (highlight != null) + highlight.prepare(); + } + + @Override + public boolean equals(Object o) { + if ( ! (o instanceof Presentation)) return false; + Presentation p = (Presentation) o; + return QueryHelper.equals(bolding, p.bolding) && QueryHelper.equals(summary, p.summary); + } + + @Override + public int hashCode() { + return QueryHelper.combineHash(bolding, summary); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java index 2fb16ef503f..115a2f6dbdc 100644 --- a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java +++ b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java @@ -7,6 +7,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.ListIterator; /** * The root node of a query tree. This is always present above the actual semantic root to ease query manipulation, @@ -179,4 +180,23 @@ public class QueryTree extends CompositeItem { } } + /** + * Returns the total number of items in this query tree. + */ + public int treeSize() { + if (isEmpty()) return 0; + return(countItemsRecursively(getItemIterator().next())); + } + + private int countItemsRecursively(Item item) { + int children = 0; + if (item instanceof CompositeItem) { + CompositeItem composite = (CompositeItem)item; + for (ListIterator<Item> i = composite.getItemIterator(); i.hasNext(); ) { + children += countItemsRecursively(i.next()); + } + } + return children + 1; + } + } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java index 90d9eab687d..f58395fd5bb 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileProperties.java @@ -87,7 +87,6 @@ public class QueryProfileProperties extends Properties { */ @Override public void set(CompoundName name, Object value, Map<String, String> context) { - // TODO: Refactor try { name = unalias(name, context); @@ -101,42 +100,8 @@ public class QueryProfileProperties extends Properties { if (runtimeReference != null && ! runtimeReference.getSecond().isOverridable(name.rest(runtimeReference.getFirst().size()), context)) return; - // Check types - if ( ! profile.getTypes().isEmpty()) { - QueryProfileType type; - QueryProfileType explicitTypeFromField = null; - for (int i = 0; i < name.size(); i++) { - if (explicitTypeFromField != null) - type = explicitTypeFromField; - else - type = profile.getType(name.first(i), context); - if (type == null) continue; - - String localName = name.get(i); - FieldDescription fieldDescription = type.getField(localName); - if (fieldDescription == null && type.isStrict()) - throw new IllegalInputException("'" + localName + "' is not declared in " + type + ", and the type is strict"); - - // TODO: In addition to strictness, check legality along the way - - if (fieldDescription != null) { - if (i == name.size() - 1) { // at the end of the path, check the assignment type - value = fieldDescription.getType().convertFrom(value, new ConversionContext(localName, - profile.getRegistry(), - embedder, - context)); - if (value == null) - throw new IllegalInputException("'" + value + "' is not a " + - fieldDescription.getType().toInstanceDescription()); - } - else if (fieldDescription.getType() instanceof QueryProfileFieldType) { - // If a type is specified, use that instead of the type implied by the name - explicitTypeFromField = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); - } - } - - } - } + if ( ! profile.getTypes().isEmpty()) + value = convertByType(name, value, context); if (value instanceof String && value.toString().startsWith("ref:")) { if (profile.getRegistry() == null) @@ -164,6 +129,53 @@ public class QueryProfileProperties extends Properties { } } + private Object convertByType(CompoundName name, Object value, Map<String, String> context) { + QueryProfileType type; + QueryProfileType explicitTypeFromField = null; + for (int i = 0; i < name.size(); i++) { + if (explicitTypeFromField != null) + type = explicitTypeFromField; + else + type = profile.getType(name.first(i), context); + if (type == null) continue; + + String localName = name.get(i); + FieldDescription fieldDescription = type.getField(localName); + if (fieldDescription == null && type.isStrict()) + throw new IllegalInputException("'" + localName + "' is not declared in " + type + ", and the type is strict"); + // TODO: In addition to strictness, check legality along the way + + if (fieldDescription != null) { + if (i == name.size() - 1) { // at the end of the path, check the assignment type + var conversionContext = new ConversionContext(localName, profile.getRegistry(), embedder, context); + var convertedValue = fieldDescription.getType().convertFrom(value, conversionContext); + if (convertedValue == null + && fieldDescription.getType() instanceof QueryProfileFieldType + && ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType() != null) { + // Try the value of the query profile itself instead + var queryProfileValueDescription = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType().getField(""); + if (queryProfileValueDescription != null) { + convertedValue = queryProfileValueDescription.getType().convertFrom(value, conversionContext); + if (convertedValue == null) + throw new IllegalInputException("'" + value + "' is neither a " + + fieldDescription.getType().toInstanceDescription() + " nor a " + + queryProfileValueDescription.getType().toInstanceDescription()); + } + } else if (convertedValue == null) + throw new IllegalInputException("'" + value + "' is not a " + + fieldDescription.getType().toInstanceDescription()); + + value = convertedValue; + } else if (fieldDescription.getType() instanceof QueryProfileFieldType) { + // If a type is specified, use that instead of the type implied by the name + explicitTypeFromField = ((QueryProfileFieldType) fieldDescription.getType()).getQueryProfileType(); + } + } + + } + return value; + } + @Override public void clearAll(CompoundName name, Map<String, String> context) { if (references == null) diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java index fb1552e549b..39ec3e1c647 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistry.java @@ -10,6 +10,11 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; +import com.yahoo.yolean.UncheckedInterruptedException; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; /** * A set of compiled query profiles. @@ -24,14 +29,47 @@ public class CompiledQueryProfileRegistry extends ComponentRegistry<CompiledQuer private final QueryProfileTypeRegistry typeRegistry; @Inject - public CompiledQueryProfileRegistry(QueryProfilesConfig config) { + public CompiledQueryProfileRegistry(QueryProfilesConfig config, Executor executor) { QueryProfileRegistry registry = QueryProfileConfigurer.createFromConfig(config); typeRegistry = registry.getTypeRegistry(); - for (QueryProfile inputProfile : registry.allComponents()) { - register(QueryProfileCompiler.compile(inputProfile, this)); + int maxConcurrent = 1; // TODO hold this one after Concurrency issue has been found: Math.max(1, (int)(Runtime.getRuntime().availableProcessors() * 0.20)); + BlockingQueue<CompiledQueryProfile> doneQ = new LinkedBlockingQueue<>(); + int started = 0; + int completed = 0; + try { + for (QueryProfile inputProfile : registry.allComponents()) { + abortIfInterrupted(); + if (started++ >= maxConcurrent) { + register(doneQ.take()); + completed++; + } + executor.execute(() -> { + Thread self = Thread.currentThread(); + int prevPriority = self.getPriority(); + try { + self.setPriority(Thread.MIN_PRIORITY); + doneQ.add(QueryProfileCompiler.compile(inputProfile, this)); + } finally { + self.setPriority(prevPriority); + } + }); + } + while (completed < started) { + register(doneQ.take()); + completed++; + } + } catch (InterruptedException e) { + throw new UncheckedInterruptedException("Interrupted while waiting for compiled query profiles", true); } } + // Query profile construction is very expensive and triggers no operations that automatically throws on interrupt + // We need to manually check the interrupt flag in case the container reconfigurer should shut down + private void abortIfInterrupted() { + if (Thread.interrupted()) + throw new UncheckedInterruptedException("Interrupted while building query profile registry", true); + } + /** Creates a compiled query profile registry with no types */ public CompiledQueryProfileRegistry() { this(QueryProfileTypeRegistry.emptyFrozen()); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java index ceeed9d9167..c3bd4f7b962 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/config/QueryProfileConfigurer.java @@ -19,6 +19,7 @@ import java.util.Set; /** * @author bratseth */ +@SuppressWarnings("removal") // TODO Vespa 8: remove public class QueryProfileConfigurer implements ConfigSubscriber.SingleSubscriber<QueryProfilesConfig> { private final ConfigSubscriber subscriber = new ConfigSubscriber(); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index 1d6f584bfaa..a0f38f61672 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -81,7 +81,7 @@ public class FieldDescription implements Comparable<FieldDescription> { /** * Creates a field description * - * @param name the name of the field + * @param name the name of the field, empty means it describes the value held by the query profile itself * @param type the type of the field represented as a string - see {@link com.yahoo.search.query.profile.types.FieldType} * @param aliases a list of aliases, never null. Aliases are not following dotted * (meaning they are global, not that they cannot contain dots) and are case insensitive. @@ -89,8 +89,6 @@ public class FieldDescription implements Comparable<FieldDescription> { * @param overridable whether this can be overridden when first set in a profile. Default: true */ public FieldDescription(CompoundName name, FieldType type, List<String> aliases, boolean mandatory, boolean overridable) { - if (name.isEmpty()) - throw new IllegalArgumentException("Illegal name ''"); for (String nameComponent : name.asList()) QueryProfile.validateName(nameComponent); this.name = name; diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java index 87f39b88981..1b036b96e01 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/PropertyAliases.java @@ -38,7 +38,6 @@ public class PropertyAliases extends Properties { */ protected CompoundName unalias(CompoundName nameOrAlias) { if (aliases.isEmpty()) return nameOrAlias; - if (nameOrAlias.size() > 1) return nameOrAlias; // aliases are simple names CompoundName properName = aliases.get(nameOrAlias.getLowerCasedName()); return (properName != null) ? properName : nameOrAlias; } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 9a09c23b23b..243915662d2 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -122,12 +122,16 @@ public class QueryProperties extends Properties { if (key.last().equals(Select.GROUPING)) return query.getSelect().getGroupingString(); } } - else if (key.size() == 2 && key.first().equals(Presentation.PRESENTATION)) { - if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); - if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); - if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat(); - if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming(); - if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields(); + else if (key.first().equals(Presentation.PRESENTATION)) { + if (key.size() == 2) { + if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); + if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); + if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat(); + if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming(); + if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields(); + } else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) { + if (key.last().equals(Presentation.TENSORS)) return query.getPresentation().getTensorShortForm(); + } } else if (key.first().equals("rankfeature") || key.first().equals("featureoverride")) { // featureoverride is deprecated return query.getRanking().getFeatures().getObject(key.rest().toString()); @@ -273,17 +277,27 @@ public class QueryProperties extends Properties { throwIllegalParameter(key.rest().toString(), Ranking.RANKING); } } - else if (key.size() == 2 && key.first().equals(Presentation.PRESENTATION)) { - if (key.last().equals(Presentation.BOLDING)) - query.getPresentation().setBolding(asBoolean(value, true)); - else if (key.last().equals(Presentation.SUMMARY)) - query.getPresentation().setSummary(asString(value, "")); - else if (key.last().equals(Presentation.FORMAT)) - query.getPresentation().setFormat(asString(value,"")); - else if (key.last().equals(Presentation.TIMING)) - query.getPresentation().setTiming(asBoolean(value, true)); - else if (key.last().equals(Presentation.SUMMARY_FIELDS)) - query.getPresentation().setSummaryFields(asString(value,"")); + else if (key.first().equals(Presentation.PRESENTATION)) { + if (key.size() == 2) { + if (key.last().equals(Presentation.BOLDING)) + query.getPresentation().setBolding(asBoolean(value, true)); + else if (key.last().equals(Presentation.SUMMARY)) + query.getPresentation().setSummary(asString(value, "")); + else if (key.last().equals(Presentation.FORMAT)) + query.getPresentation().setFormat(asString(value, "")); + else if (key.last().equals(Presentation.TIMING)) + query.getPresentation().setTiming(asBoolean(value, true)); + else if (key.last().equals(Presentation.SUMMARY_FIELDS)) + query.getPresentation().setSummaryFields(asString(value, "")); + else + throwIllegalParameter(key.last(), Presentation.PRESENTATION); + } + else if (key.size() == 3 && key.get(1).equals(Presentation.FORMAT)) { + if (key.last().equals(Presentation.TENSORS)) + query.getPresentation().setTensorShortForm(asString(value, "")); + else + throwIllegalParameter(key.last(), Presentation.FORMAT); + } else throwIllegalParameter(key.last(), Presentation.PRESENTATION); } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java index 83daf6398c3..fff8935eefb 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/DefaultPositionSearcher.java @@ -5,17 +5,20 @@ import static com.yahoo.prelude.searcher.PosSearcher.POSITION_PARSING; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Location; +import com.yahoo.prelude.query.GeoLocationItem; import com.yahoo.search.Query; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; +import com.google.inject.Inject; import java.util.List; /** - * If default position has not been set, it will be set here. + * If position attribute has not been set, it will be set here. * * @author baldersheim */ @@ -23,6 +26,17 @@ import java.util.List; @Before(PhaseNames.TRANSFORMED_QUERY) public class DefaultPositionSearcher extends Searcher { + private final boolean useV8GeoPositions; + + @Inject + public DefaultPositionSearcher(DocumentmanagerConfig cfg) { + this.useV8GeoPositions = cfg.usev8geopositions(); + } + + DefaultPositionSearcher() { + this.useV8GeoPositions = false; + } + @Override public com.yahoo.search.Result search(Query query, Execution execution) { Location location = query.getRanking().getLocation(); @@ -40,6 +54,12 @@ public class DefaultPositionSearcher extends Searcher { location.setAttribute(facts.getDefaultPosition(null)); } } + if (useV8GeoPositions && (location != null) && (location.getAttribute() != null)) { + var geoLoc = new GeoLocationItem(location); + query.getModel().getQueryTree().and(geoLoc); + location = null; + query.getRanking().setLocation(location); + } return execution.search(query); } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index f9a91558254..2ba507a83b8 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -76,10 +76,10 @@ import static com.fasterxml.jackson.databind.SerializationFeature.FLUSH_AFTER_WR // NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc. public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { - private static final CompoundName WRAP_ALL_MAPS = new CompoundName("renderer.json.jsonMaps"); + private static final CompoundName WRAP_DEEP_MAPS = new CompoundName("renderer.json.jsonMaps"); + private static final CompoundName WRAP_WSETS = new CompoundName("renderer.json.jsonWsets"); private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug"); private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback"); - private static final CompoundName TENSOR_FORMAT = new CompoundName("format.tensors"); // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString private static final String BUCKET_LIMITS = "limits"; @@ -125,18 +125,46 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private JsonGenerator generator; private FieldConsumer fieldConsumer; private Deque<Integer> renderedChildren; - private boolean debugRendering; - private boolean jsonMaps; + static class FieldConsumerSettings { + boolean debugRendering = false; + boolean jsonDeepMaps = false; + boolean jsonWsets = false; + boolean jsonMapsAll = false; + boolean jsonWsetsAll = false; + boolean tensorShortForm = false; + boolean convertDeep() { return (jsonDeepMaps || jsonWsets); } + void init() { + this.debugRendering = false; + this.jsonDeepMaps = false; + this.jsonWsets = false; + this.jsonMapsAll = false; + this.jsonWsetsAll = false; + this.tensorShortForm = false; + } + void getSettings(Query q) { + if (q == null) { + init(); + return; + } + var props = q.properties(); + this.debugRendering = props.getBoolean(DEBUG_RENDERING_KEY, false); + this.jsonDeepMaps = props.getBoolean(WRAP_DEEP_MAPS, false); + this.jsonWsets = props.getBoolean(WRAP_WSETS, false); + // we may need more fine tuning, but for now use the same query parameters here: + this.jsonMapsAll = props.getBoolean(WRAP_DEEP_MAPS, false); + this.jsonWsetsAll = props.getBoolean(WRAP_WSETS, false); + this.tensorShortForm = q.getPresentation().getTensorShortForm(); + } + } + private final FieldConsumerSettings fieldConsumerSettings = new FieldConsumerSettings(); private LongSupplier timeSource; private OutputStream stream; - private boolean tensorShortFormRendering = false; - public JsonRenderer() { this(null); } - /** + /** * Creates a json renderer using a custom executor. * Using a custom executor is useful for tests to avoid creating new threads for each renderer registry. */ @@ -151,8 +179,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { * method will be invoked when creating the first renderer instance, but not * for each fresh clone used by individual results. * + * @deprecated Will be removed in Vespa 8. Override the individual render methods of {@link JsonRenderer} to alter + * rendering behaviour. Override {@link #createFieldConsumer(boolean)} and sub-class {@link FieldConsumer} + * to alter rendering of hit fields. * @return an object mapper for the internal JsonFactory */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 make private protected static ObjectMapper createJsonCodec() { return new ObjectMapper().disable(FLUSH_AFTER_WRITE_VALUE); } @@ -160,9 +192,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void init() { super.init(); - debugRendering = false; - jsonMaps = false; - setGenerator(null, debugRendering); + fieldConsumerSettings.init(); + setGenerator(null, fieldConsumerSettings); renderedChildren = null; timeSource = System::currentTimeMillis; stream = null; @@ -171,10 +202,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void beginResponse(OutputStream stream) throws IOException { beginJsonCallback(stream); - debugRendering = getDebugRendering(getResult().getQuery()); - jsonMaps = getWrapAllMaps(getResult().getQuery()); - tensorShortFormRendering = getTensorShortFormRendering(getResult().getQuery()); - setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering); + fieldConsumerSettings.getSettings(getResult().getQuery()); + setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), fieldConsumerSettings); renderedChildren = new ArrayDeque<>(); generator.writeStartObject(); renderTrace(getExecution().trace()); @@ -204,20 +233,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeEndObject(); } - private boolean getWrapAllMaps(Query q) { - return q != null && q.properties().getBoolean(WRAP_ALL_MAPS, false); - } - - private boolean getDebugRendering(Query q) { - return q != null && q.properties().getBoolean(DEBUG_RENDERING_KEY, false); - } - - private boolean getTensorShortFormRendering(Query q) { - if (q == null || q.properties().get(TENSOR_FORMAT) == null) - return false; - return q.properties().getString(TENSOR_FORMAT).equalsIgnoreCase("short"); - } - protected void renderTrace(Trace trace) throws IOException { if (!trace.traceNode().children().iterator().hasNext()) return; if (getResult().getQuery().getTraceLevel() == 0) return; @@ -520,17 +535,26 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return null; } - private void setGenerator(JsonGenerator generator, boolean debugRendering) { + private void setGenerator(JsonGenerator generator, FieldConsumerSettings settings) { this.generator = generator; - this.fieldConsumer = generator == null ? null : createFieldConsumer(generator, debugRendering, jsonMaps); + this.fieldConsumer = generator == null ? null : createFieldConsumer(generator, settings); } + /** Override this method to use a custom {@link FieldConsumer} sub-class to render fields */ + protected FieldConsumer createFieldConsumer(boolean debugRendering) { + fieldConsumerSettings.debugRendering = debugRendering; + return createFieldConsumer(generator, fieldConsumerSettings); + } + + /** @deprecated Will be removed in Vespa 8. Use {@link #createFieldConsumer(boolean)} instead. */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove method protected FieldConsumer createFieldConsumer(JsonGenerator generator, boolean debugRendering) { - return createFieldConsumer(generator, debugRendering, this.jsonMaps); + fieldConsumerSettings.debugRendering = debugRendering; + return createFieldConsumer(generator, fieldConsumerSettings); } - private FieldConsumer createFieldConsumer(JsonGenerator generator, boolean debugRendering, boolean jsonMaps) { - return new FieldConsumer(generator, debugRendering, tensorShortFormRendering, jsonMaps); + private FieldConsumer createFieldConsumer(JsonGenerator generator, FieldConsumerSettings settings) { + return new FieldConsumer(generator, settings); } /** @@ -548,23 +572,39 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { public static class FieldConsumer implements Hit.RawUtf8Consumer, TraceRenderer.FieldConsumer { private final JsonGenerator generator; - private final boolean debugRendering; - private final boolean jsonMaps; - private final boolean tensorShortForm; - + private final FieldConsumerSettings settings; private MutableBoolean hasFieldsField; + /** @deprecated Will be removed in Vespa 8. Use {@link #FieldConsumer(boolean, boolean, boolean)} instead. */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 Remove public FieldConsumer(JsonGenerator generator, boolean debugRendering) { this(generator, debugRendering, false); } + + /** @deprecated Will be removed in Vespa 8. Use {@link #FieldConsumer(boolean, boolean, boolean)} instead. */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 Remove public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm) { this(generator, debugRendering, tensorShortForm, false); } + + /** Invoke this from your constructor when sub-classing {@link FieldConsumer} */ + protected FieldConsumer(boolean debugRendering, boolean tensorShortForm, boolean jsonMaps) { + this(null, debugRendering, tensorShortForm, jsonMaps); + } + + /** @deprecated Will be removed in Vespa 8. Use {@link #FieldConsumer(boolean, boolean, boolean)} instead. */ + @Deprecated(forRemoval = true, since = "7") // TODO Vespa 8 remove public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm, boolean jsonMaps) { this.generator = generator; - this.debugRendering = debugRendering; - this.tensorShortForm = tensorShortForm; - this.jsonMaps = jsonMaps; + this.settings = new FieldConsumerSettings(); + this.settings.debugRendering = debugRendering; + this.settings.tensorShortForm = tensorShortForm; + this.settings.jsonDeepMaps = jsonMaps; + } + + FieldConsumer(JsonGenerator generator, FieldConsumerSettings settings) { + this.generator = generator; + this.settings = settings; } /** @@ -578,14 +618,14 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { /** Call before rendering a field to the generator */ void ensureFieldsField() throws IOException { if (hasFieldsField.get()) return; - generator.writeObjectFieldStart(FIELDS); + generator().writeObjectFieldStart(FIELDS); hasFieldsField.set(true); } /** Call after all fields in a hit to close the "fields" field of the JSON object */ void endHitFields() throws IOException { if ( ! hasFieldsField.get()) return; - generator.writeEndObject(); + generator().writeEndObject(); this.hasFieldsField = null; } @@ -594,7 +634,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { try { if (shouldRender(name, value)) { ensureFieldsField(); - generator.writeFieldName(name); + generator().writeFieldName(name); renderFieldContents(value); } } @@ -608,8 +648,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { try { if (shouldRenderUtf8Value(name, length)) { ensureFieldsField(); - generator.writeFieldName(name); - generator.writeUTF8String(utf8Data, offset, length); + generator().writeFieldName(name); + generator().writeUTF8String(utf8Data, offset, length); } } catch (IOException e) { @@ -618,7 +658,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } protected boolean shouldRender(String name, Object value) { - if (debugRendering) return true; + if (settings.debugRendering) return true; if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; if (value instanceof CharSequence && ((CharSequence) value).length() == 0) return false; // StringFieldValue cannot hold a null, so checking length directly is OK: @@ -628,47 +668,68 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } protected boolean shouldRenderUtf8Value(String name, int length) { - if (debugRendering) return true; + if (settings.debugRendering) return true; if (name.startsWith(VESPA_HIDDEN_FIELD_PREFIX)) return false; if (length == 0) return false; return true; } - private static Inspector deepWrapAsMap(Inspector data) { - if (data.type() == Type.ARRAY) { - var map = new Value.ObjectValue(); - for (int i = 0; i < data.entryCount(); i++) { - Inspector obj = data.entry(i); - if (map != null && obj.type() == Type.OBJECT && obj.fieldCount() == 2) { - Inspector key = obj.field("key"); - Inspector value = obj.field("value"); - if (key.type() == Type.STRING && value.valid()) { - map.put(key.asString(), deepWrapAsMap(value)); - } else { - map = null; - } - } else { - map = null; - } + private Inspector maybeConvertMap(Inspector data) { + var map = new Value.ObjectValue(); + for (int i = 0; i < data.entryCount(); i++) { + Inspector obj = data.entry(i); + if (obj.type() != Type.OBJECT || obj.fieldCount() != 2) { + return null; + } + Inspector key = obj.field("key"); + Inspector value = obj.field("value"); + if (! key.valid()) return null; + if (! value.valid()) return null; + if (key.type() != Type.STRING && !settings.jsonMapsAll) { + return null; } - if (map != null) { - return map; + if (settings.convertDeep()) { + value = deepMaybeConvert(value); } - var array = new Value.ArrayValue(); - for (int i = 0; i < data.entryCount(); i++) { - Inspector obj = data.entry(i); - array.add(deepWrapAsMap(obj)); + if (key.type() == Type.STRING) { + map.put(key.asString(), value); + } else { + map.put(key.toString(), value); } - return array; } - if (data.type() == Type.OBJECT) { - var object = new Value.ObjectValue(); - for (var entry : data.fields()) { - object.put(entry.getKey(), deepWrapAsMap(entry.getValue())); + return map; + } + + private Inspector maybeConvertWset(Inspector data) { + var wset = new Value.ObjectValue(); + for (int i = 0; i < data.entryCount(); i++) { + Inspector obj = data.entry(i); + if (obj.type() != Type.OBJECT || obj.fieldCount() != 2) { + return null; + } + Inspector item = obj.field("item"); + Inspector weight = obj.field("weight"); + if (! item.valid()) return null; + if (! weight.valid()) return null; + // TODO support non-integer weights? + if (weight.type() != Type.LONG) return null; + if (item.type() == Type.STRING) { + wset.put(item.asString(), weight.asLong()); + } else if (settings.jsonWsetsAll) { + wset.put(item.toString(), weight.asLong()); + } else { + return null; } - return object; } - return data; + return wset; + } + + private Inspector convertInsideObject(Inspector data) { + var object = new Value.ObjectValue(); + for (var entry : data.fields()) { + object.put(entry.getKey(), deepMaybeConvert(entry.getValue())); + } + return object; } private static Inspector wrapAsMap(Inspector data) { @@ -688,19 +749,61 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return map; } - private void renderInspector(Inspector data) throws IOException { - Inspector asMap = jsonMaps ? deepWrapAsMap(data) : wrapAsMap(data); - if (asMap != null) { - renderInspectorDirect(asMap); - } else { - renderInspectorDirect(data); + private Inspector deepMaybeConvert(Inspector data) { + if (data.type() == Type.ARRAY) { + if (settings.jsonDeepMaps) { + var map = maybeConvertMap(data); + if (map != null) return map; + } + if (settings.jsonWsets) { + var wset = maybeConvertWset(data); + if (wset != null) return wset; + } + } + if (data.type() == Type.OBJECT) { + return convertInsideObject(data); + } + return data; + } + + private Inspector convertTopLevelArray(Inspector data) { + if (data.entryCount() > 0) { + var map = maybeConvertMap(data); + if (map != null) return map; + if (settings.jsonWsets) { + var wset = maybeConvertWset(data); + if (wset != null) return wset; + } + if (settings.convertDeep()) { + var array = new Value.ArrayValue(); + for (int i = 0; i < data.entryCount(); i++) { + Inspector obj = data.entry(i); + array.add(deepMaybeConvert(obj)); + } + return array; + } } + return data; + } + + private Inspector maybeConvertData(Inspector data) throws IOException { + if (data.type() == Type.ARRAY) { + return convertTopLevelArray(data); + } + if (settings.convertDeep() && data.type() == Type.OBJECT) { + return convertInsideObject(data); + } + return data; + } + + private void renderInspector(Inspector data) throws IOException { + renderInspectorDirect(maybeConvertData(data)); } private void renderInspectorDirect(Inspector data) throws IOException { StringBuilder intermediate = new StringBuilder(); JsonRender.render(data, intermediate, true); - generator.writeRawValue(intermediate.toString()); + generator().writeRawValue(intermediate.toString()); } protected void renderFieldContents(Object field) throws IOException { @@ -714,68 +817,75 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { @Override public void accept(Object field) throws IOException { if (field == null) { - generator.writeNull(); + generator().writeNull(); } else if (field instanceof Boolean) { - generator.writeBoolean((Boolean)field); + generator().writeBoolean((Boolean)field); } else if (field instanceof Number) { renderNumberField((Number) field); } else if (field instanceof TreeNode) { - generator.writeTree((TreeNode) field); + generator().writeTree((TreeNode) field); } else if (field instanceof Tensor) { renderTensor(Optional.of((Tensor)field)); } else if (field instanceof FeatureData) { - generator.writeRawValue(((FeatureData)field).toJson(tensorShortForm)); + generator().writeRawValue(((FeatureData)field).toJson(settings.tensorShortForm)); } else if (field instanceof Inspectable) { renderInspectorDirect(((Inspectable)field).inspect()); } else if (field instanceof JsonProducer) { - generator.writeRawValue(((JsonProducer) field).toJson()); + generator().writeRawValue(((JsonProducer) field).toJson()); } else if (field instanceof StringFieldValue) { - generator.writeString(((StringFieldValue)field).getString()); + generator().writeString(((StringFieldValue)field).getString()); } else if (field instanceof TensorFieldValue) { renderTensor(((TensorFieldValue)field).getTensor()); } else if (field instanceof FieldValue) { // the null below is the field which has already been written ((FieldValue) field).serialize(null, new JsonWriter(generator)); } else { - generator.writeString(field.toString()); + generator().writeString(field.toString()); } } private void renderNumberField(Number field) throws IOException { if (field instanceof Integer) { - generator.writeNumber(field.intValue()); + generator().writeNumber(field.intValue()); } else if (field instanceof Float) { - generator.writeNumber(field.floatValue()); + generator().writeNumber(field.floatValue()); } else if (field instanceof Double) { - generator.writeNumber(field.doubleValue()); + generator().writeNumber(field.doubleValue()); } else if (field instanceof Long) { - generator.writeNumber(field.longValue()); + generator().writeNumber(field.longValue()); } else if (field instanceof Byte || field instanceof Short) { - generator.writeNumber(field.intValue()); + generator().writeNumber(field.intValue()); } else if (field instanceof BigInteger) { - generator.writeNumber((BigInteger) field); + generator().writeNumber((BigInteger) field); } else if (field instanceof BigDecimal) { - generator.writeNumber((BigDecimal) field); + generator().writeNumber((BigDecimal) field); } else { - generator.writeNumber(field.doubleValue()); + generator().writeNumber(field.doubleValue()); } } private void renderTensor(Optional<Tensor> tensor) throws IOException { if (tensor.isEmpty()) { - generator.writeStartObject(); - generator.writeArrayFieldStart("cells"); - generator.writeEndArray(); - generator.writeEndObject(); + generator().writeStartObject(); + generator().writeArrayFieldStart("cells"); + generator().writeEndArray(); + generator().writeEndObject(); return; } - if (tensorShortForm) { - generator.writeRawValue(new String(JsonFormat.encodeShortForm(tensor.get()), StandardCharsets.UTF_8)); + if (settings.tensorShortForm) { + generator().writeRawValue(new String(JsonFormat.encodeShortForm(tensor.get()), StandardCharsets.UTF_8)); } else { - generator.writeRawValue(new String(JsonFormat.encode(tensor.get()), StandardCharsets.UTF_8)); + generator().writeRawValue(new String(JsonFormat.encode(tensor.get()), StandardCharsets.UTF_8)); } } + private JsonGenerator generator() { + if (generator == null) + throw new UnsupportedOperationException("Generator required but not assigned. " + + "All accept() methods must be overridden when sub-classing FieldConsumer"); + return generator; + } + } } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java index c507069b948..9374027504e 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/Execution.java @@ -8,6 +8,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.language.process.SpecialTokenRegistry; +import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.Processor; import com.yahoo.processing.Request; import com.yahoo.processing.Response; @@ -18,7 +19,6 @@ import com.yahoo.search.cluster.PingableSearcher; import com.yahoo.search.rendering.Renderer; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.search.statistics.TimeTracker; - import java.util.Objects; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -41,6 +41,8 @@ import java.util.concurrent.Executors; */ public class Execution extends com.yahoo.processing.execution.Execution { + /** @deprecated - applications should define their own summary class instead */ + @Deprecated(since = "7", forRemoval = true) public static final String ATTRIBUTEPREFETCH = "attributeprefetch"; /** @@ -532,18 +534,23 @@ public class Execution extends com.yahoo.processing.execution.Execution { } /** - * Fill hit properties with values from all in-memory attributes. - * This can be done with good performance on many more hits than - * those for which fill is called with the final summary class, so - * if filtering can be done using only in-memory attribute data, - * this method should be preferred over {@link #fill} to get that data for filtering. - * <p> - * Calling this on already filled results has no cost. + * Fill hit properties with values from some in-memory attributes. + * Not all attributes are included, and *which* attributes are + * subject to change depending on what Vespa needs internally. + * + * Applications should prefer to define their own summary class + * with only the in-memory attributes they need, and call + * fill(result, "foo") with the name of their own summary class + * instead of "foo". * + * @deprecated use fill(Result, String) + * + * TODO Remove on Vespa 9. * @param result the result to fill */ + @Deprecated public void fillAttributes(Result result) { - fill(result, ATTRIBUTEPREFETCH); + fill(result, VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } /** diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index 13a9f9510cd..2dfa6ef3e3a 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -12,6 +12,7 @@ import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig.Documentdb.Summaryclass.Fields; +import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -95,7 +96,7 @@ public class FieldFiller extends Searcher { for (Documentdb db : config.documentdb()) { for (Summaryclass summary : db.summaryclass()) { Set<String> attributes; - if (Execution.ATTRIBUTEPREFETCH.equals(summary.name())) { + if (SORTABLE_ATTRIBUTES_SUMMARY_CLASS.equals(summary.name())) { attributes = new HashSet<>(summary.fields().size()); for (Fields f : summary.fields()) { attributes.add(f.name()); @@ -134,11 +135,11 @@ public class FieldFiller extends Searcher { } if (intersectionOfAttributes.containsAll(summaryFields)) { - if ( ! Execution.ATTRIBUTEPREFETCH.equals(summaryClass)) { - execution.fill(result, Execution.ATTRIBUTEPREFETCH); + if (! SORTABLE_ATTRIBUTES_SUMMARY_CLASS.equals(summaryClass)) { + execution.fill(result, SORTABLE_ATTRIBUTES_SUMMARY_CLASS); } } else { - // Yes, summaryClass may be Execution.ATTRIBUTEPREFETCH here + // Yes, summaryClass may be SORTABLE_ATTRIBUTES_SUMMARY_CLASS here if ( ! summaryDb.hasAll(summaryFields, summaryClass, result.getQuery().getModel().getRestrict())) { execution.fill(result, null); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index c0c5b0ee0b0..846f70fd26b 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -400,7 +400,7 @@ public class YqlParser implements Parser { case NON_EMPTY: return ensureNonEmpty(ast); default: - throw newUnexpectedArgumentException(names.get(0), DOT_PRODUCT, NEAREST_NEIGHBOR, + throw newUnexpectedArgumentException(names.get(0), DOT_PRODUCT, GEO_LOCATION, NEAREST_NEIGHBOR, RANGE, RANK, USER_QUERY, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE, USER_INPUT, NON_EMPTY); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java index ab9da8ccee5..f20fc85b448 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java @@ -20,6 +20,7 @@ import static com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher.STREAMING_S /** * Generates mail-specific query metrics. */ +@SuppressWarnings("removal") // TODO Vespa 8: remove (com.yahoo.log.event) public class MetricsSearcher extends Searcher { private static final CompoundName metricsearcherId=new CompoundName("metricsearcher.id"); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java new file mode 100644 index 00000000000..e252a230d4f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/QueryEncoder.java @@ -0,0 +1,90 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.streamingvisitors; + +import com.yahoo.fs4.MapEncoder; +import com.yahoo.prelude.fastsearch.DocumentDatabase; +import com.yahoo.prelude.query.Highlight; +import com.yahoo.search.Query; +import com.yahoo.search.dispatch.rpc.ProtobufSerialization; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Encodes the query in binary form. + * + * @author bratseth + */ +class QueryEncoder { + + /** + * Encodes properties of this query. + * + * @param buffer the buffer to encode to + * @param encodeQueryData true to encode all properties, false to only include session information, not actual query data + * @return the encoded length + */ + static int encodeAsProperties(Query query, ByteBuffer buffer, boolean encodeQueryData) { + // Make sure we don't encode anything here if we have turned the property feature off + // Due to sendQuery we sometimes end up turning this feature on and then encoding a 0 int as the number of + // property maps - that's ok (probably we should simplify by just always turning the feature on) + if (! hasEncodableProperties(query)) return 0; + + int start = buffer.position(); + int mapCountPosition = buffer.position(); + buffer.putInt(0); // map count will go here + int mapCount = 0; + mapCount += query.getRanking().getProperties().encode(buffer, encodeQueryData); + if (encodeQueryData) { + mapCount += query.getRanking().getFeatures().encode(buffer); + if (query.getPresentation().getHighlight() != null) { + mapCount += MapEncoder.encodeMultiMap(Highlight.HIGHLIGHTTERMS, + query.getPresentation().getHighlight().getHighlightTerms(), buffer); + } + mapCount += MapEncoder.encodeMap("model", createModelMap(query), buffer); + } + mapCount += MapEncoder.encodeSingleValue(DocumentDatabase.MATCH_PROPERTY, DocumentDatabase.SEARCH_DOC_TYPE_KEY, + query.getModel().getDocumentDb(), buffer); + mapCount += MapEncoder.encodeMap("caches", createCacheSettingMap(query), buffer); + buffer.putInt(mapCountPosition, mapCount); + return buffer.position() - start; + } + + static boolean hasEncodableProperties(Query query) { + if ( ! query.getRanking().getProperties().isEmpty()) return true; + if ( ! query.getRanking().getFeatures().isEmpty()) return true; + if ( query.getRanking().getFreshness() != null) return true; + if ( query.getModel().getSearchPath() != null) return true; + if ( query.getModel().getDocumentDb() != null) return true; + if ( query.getPresentation().getHighlight() != null && + ! query.getPresentation().getHighlight().getHighlightItems().isEmpty()) return true; + return false; + } + + private static Map<String, Boolean> createCacheSettingMap(Query query) { + if (query.getGroupingSessionCache() && query.getRanking().getQueryCache()) { + Map<String, Boolean> cacheSettingMap = new HashMap<>(); + cacheSettingMap.put("grouping", true); + cacheSettingMap.put("query", true); + return cacheSettingMap; + } + if (query.getGroupingSessionCache()) + return Collections.singletonMap("grouping", true); + if (query.getRanking().getQueryCache()) + return Collections.singletonMap("query", true); + return Collections.emptyMap(); + } + + private static Map<String, String> createModelMap(Query query) { + Map<String, String> m = new HashMap<>(); + if (query.getModel().getSearchPath() != null) m.put("searchpath", query.getModel().getSearchPath()); + + int traceLevel = ProtobufSerialization.getTraceLevelForBackend(query); + if (traceLevel > 0) m.put("tracelevel", String.valueOf(traceLevel)); + + return m; + } + +} diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index e2233d51ae4..b2e4821f164 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -3,13 +3,10 @@ package com.yahoo.vespa.streamingvisitors; import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.AckToken; -import com.yahoo.documentapi.DocumentAccess; import com.yahoo.documentapi.VisitorControlHandler; import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; -import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; -import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; @@ -41,7 +38,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; /** @@ -187,7 +183,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { params.setLibraryParameter("location", af); } - if (query.hasEncodableProperties()) { + if (QueryEncoder.hasEncodableProperties(query)) { encodeQueryData(query, 1, ed); params.setLibraryParameter("rankproperties", ed.getEncodedData()); } @@ -254,7 +250,7 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { ed.setReturned(query.getModel().getQueryTree().getRoot().encode(buf)); break; case 1: - ed.setReturned(query.encodeAsProperties(buf, true)); + ed.setReturned(QueryEncoder.encodeAsProperties(query, buf, true)); break; case 2: throw new IllegalArgumentException("old aggregation no longer exists!"); diff --git a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj index 46117374e59..39ea6435393 100644 --- a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj +++ b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj @@ -69,6 +69,7 @@ TOKEN : <EQUALS: "="> | <EXCLAMATION: "!"> | <INCLUDEDIRECTIVE: "@include"> | + <LANGUAGEDIRECTIVE: "@language"> | <LARGER: ">"> | <LARGEREQUALS: ">="> | <LEFTBRACE: "("> | @@ -86,8 +87,8 @@ TOKEN : <SLASH: "/"> | <SMALLER: "<"> | <SMALLEREQUALS: "<="> | + <STAR: "*"> | <STEMMINGDIRECTIVE: "@stemming"> | - <LANGUAGEDIRECTIVE: "@language"> | <SUPERDIRECTIVE: "@super"> | <IDENTIFIER: (~[ "\u0000"-"\u002f","\u003a"-"\u003f","\u005b"-"\u005d","\u007b"-"\u00a7","\u00a9","\u00ab"-"\u00ae","\u00b0"-"\u00b3","\u00b6"-"\u00b7","\u00b9","\u00bb"-"\u00bf", @@ -312,10 +313,14 @@ NamespaceProduction namespaceProduction() : ReferenceTermProduction referenceTermProduction() : { String reference; + boolean produceAll = false; } { - <LEFTSQUAREBRACKET> reference = referenceIdentifier() <RIGHTSQUAREBRACKET> - { return new ReferenceTermProduction(reference); } + <LEFTSQUAREBRACKET> + reference = referenceIdentifier() + (<STAR> { produceAll = true; })? + <RIGHTSQUAREBRACKET> + { return new ReferenceTermProduction(reference, produceAll); } } LiteralTermProduction literalTermProduction() : diff --git a/container-search/src/main/resources/configdefinitions/search.config.qr-start.def b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def index e2856e137f0..c58f9944d61 100644 --- a/container-search/src/main/resources/configdefinitions/search.config.qr-start.def +++ b/container-search/src/main/resources/configdefinitions/search.config.qr-start.def @@ -24,7 +24,7 @@ jvm.stacksize int default=512 restart jvm.compressedClassSpaceSize int default=32 restart ## Base value of maximum direct memory size (in megabytes) -jvm.baseMaxDirectMemorySize int default=75 restart +jvm.baseMaxDirectMemorySize int default=16 restart ## Amount of direct memory used for caching. (in megabytes) jvm.directMemorySizeCache int default=0 restart diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java index 11424cc7e4e..f0c29e64839 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java @@ -490,6 +490,18 @@ public class QueryCanonicalizerTestCase { assertFalse(shoe.usePositionData()); } + @Test + public void queryTreeExceedsAllowedSize() { + Query query = new Query(); + QueryTree tree = query.getModel().getQueryTree(); + tree.setRoot(new WordItem("A")); + tree.and(new WordItem("B")); + + assertNull(QueryCanonicalizer.canonicalize(query)); + query.properties().set("maxQueryItems", 2); + assertEquals("Query tree exceeds allowed item count. Configured limit: 2 - Item count: 3", QueryCanonicalizer.canonicalize(query)); + } + private void assertCanonicalized(String canonicalForm, String expectedError, Item root) { Query query = new Query(); query.getModel().getQueryTree().setRoot(root); diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java index 2a7e9da2992..ef3526f2fb1 100644 --- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/QuotingSearcherTestCase.java @@ -32,7 +32,6 @@ import static org.junit.Assert.assertTrue; * * @author Steinar Knutsen */ -@SuppressWarnings("deprecation") public class QuotingSearcherTestCase { public static QuotingSearcher createQuotingSearcher(String configId) { diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java index eb69372c22b..17eb4120b84 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java @@ -29,7 +29,7 @@ public class ConditionTestCase { var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); TermCondition term = new TermCondition("foo", linguistics); Query query = new Query("?query=foo"); - assertTrue(term.matches(new Evaluation(query).freshRuleEvaluation())); + assertTrue(term.matches(new Evaluation(query, null).freshRuleEvaluation())); } @Test @@ -41,11 +41,11 @@ public class ConditionTestCase { sequence.addCondition(term1); sequence.addCondition(term2); Query query = new Query("?query=foo+bar"); - assertTrue(query + " matches " + sequence,sequence.matches(new Evaluation(query).freshRuleEvaluation())); + assertTrue(query + " matches " + sequence,sequence.matches(new Evaluation(query, null).freshRuleEvaluation())); Query query2 = new Query("?query=foo"); - assertFalse(query2 + " does not match " + sequence,sequence.matches(new Evaluation(query2).freshRuleEvaluation())); + assertFalse(query2 + " does not match " + sequence,sequence.matches(new Evaluation(query2, null).freshRuleEvaluation())); Query query3 = new Query("?query=bar"); - assertFalse(query3 + " does not match " + sequence,sequence.matches(new Evaluation(query3).freshRuleEvaluation())); + assertFalse(query3 + " does not match " + sequence,sequence.matches(new Evaluation(query3, null).freshRuleEvaluation())); } @Test @@ -57,11 +57,11 @@ public class ConditionTestCase { choice.addCondition(term1); choice.addCondition(term2); Query query1 = new Query("?query=foo+bar"); - assertTrue(query1 + " matches " + choice, choice.matches(new Evaluation(query1).freshRuleEvaluation())); + assertTrue(query1 + " matches " + choice, choice.matches(new Evaluation(query1, null).freshRuleEvaluation())); Query query2 = new Query("?query=foo"); - assertTrue(query2 + " matches " + choice, choice.matches(new Evaluation(query2).freshRuleEvaluation())); + assertTrue(query2 + " matches " + choice, choice.matches(new Evaluation(query2, null).freshRuleEvaluation())); Query query3 = new Query("?query=bar"); - assertTrue(query3 + " matches " + choice, choice.matches(new Evaluation(query3).freshRuleEvaluation())); + assertTrue(query3 + " matches " + choice, choice.matches(new Evaluation(query3, null).freshRuleEvaluation())); } @Test @@ -75,13 +75,13 @@ public class ConditionTestCase { ProductionRule rule = new ReplacingProductionRule(); rule.setCondition(reference); rule.setProduction(new ProductionList()); - RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); + RuleBase ruleBase = new RuleBase("test"); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); Query query = new Query("?query=foo"); - assertTrue(query + " matches " + reference,reference.matches(new Evaluation(query).freshRuleEvaluation())); + assertTrue(query + " matches " + reference,reference.matches(new Evaluation(query, null).freshRuleEvaluation())); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java index ca78cae70ea..c398f0ed99e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MatchOnlyIfNotOnlyTermTestCase.java @@ -17,8 +17,8 @@ public class MatchOnlyIfNotOnlyTermTestCase extends RuleBaseAbstractTestCase { @Test public void testMatch() { - assertSemantics("RANK (AND justin timberlake) showname:\"saturday night live\"!1000", "justin timberlake snl"); - assertSemantics("RANK (AND justin timberlake) showname:\"saturday night live\"!1000", "justin timberlake saturday night live"); + assertSemantics("RANK showname:\"saturday night live\"!1000 (AND justin timberlake)", "justin timberlake snl"); + assertSemantics("RANK showname:\"saturday night live\"!1000 (AND justin timberlake)", "justin timberlake saturday night live"); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java index 3051dd77190..1d6eea8d7de 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/OrPhraseTestCase.java @@ -14,7 +14,7 @@ public class OrPhraseTestCase extends RuleBaseAbstractTestCase { @Test public void testReplacing1() { - assertSemantics("OR (AND new york) title:\"software engineer\"","software engineer new york"); + assertSemantics("OR title:\"software engineer\" (AND new york)","software engineer new york"); assertSemantics("title:\"software engineer\"","software engineer"); // Skip or when there is nothing else } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java index b91e9441a2b..bafb2cb6a73 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java @@ -32,7 +32,7 @@ public class ProductionRuleTestCase { NamedCondition named = new NamedCondition("brand", term); ConditionReference reference = new ConditionReference("brand"); - TermProduction termProduction = new ReferenceTermProduction("brand", "brand"); + TermProduction termProduction = new ReferenceTermProduction("brand", "brand", false); ProductionList productionList = new ProductionList(); productionList.addProduction(termProduction); @@ -41,7 +41,7 @@ public class ProductionRuleTestCase { rule.setProduction(productionList); // To initialize the condition reference... - RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); + RuleBase ruleBase = new RuleBase("test"); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); @@ -49,7 +49,7 @@ public class ProductionRuleTestCase { assertTrue("Brand is referenced", rule.matchReferences().contains("brand")); Query query = new Query("?query=sony"); - RuleEvaluation e = new Evaluation(query).freshRuleEvaluation(); + RuleEvaluation e = new Evaluation(query, null).freshRuleEvaluation(); assertTrue(rule.matches(e)); rule.produce(e); assertEquals("AND brand:sony", query.getModel().getQueryTree().getRoot().toString()); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java index 76b2d3991c1..bee65db4347 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java @@ -128,12 +128,12 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { @Test public void testTypeChange() { - assertSemantics("RANK doors default:typechange","typechange doors"); + assertSemantics("RANK default:typechange doors","typechange doors"); } @Test public void testTypeChangeWithSingularToPluralButNonReplaceWillNotSingularify() { - assertSemantics("RANK door default:typechange","typechange door"); + assertSemantics("RANK default:typechange door","typechange door"); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java new file mode 100644 index 00000000000..8f69be2f710 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SynonymTestCase.java @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import org.junit.Test; + +/** + * @author bratseth + */ +public class SynonymTestCase { + + @Test + public void testReplacingBySynonyms() { + var tester = new RuleBaseTester("synonyms.sr"); + tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:foo"); + tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:bar"); + tester.assertSemantics("EQUIV index1:foo index1:baz index1:bar", "index1:baz"); + tester.assertSemantics("EQUIV index1:word index1:\"a phrase\"", "index1:word"); + tester.assertSemantics("EQUIV index1:word index1:\"a phrase\"", "index1:a index1:phrase"); + tester.assertSemantics("index1:other", "index1:other"); + } + + @Test + public void testAddingSynonyms() { + var tester = new RuleBaseTester("synonyms.sr"); + tester.assertSemantics("EQUIV index2:foo index2:baz index2:bar", "index2:foo"); + tester.assertSemantics("EQUIV index2:bar index2:foo index2:baz", "index2:bar"); + tester.assertSemantics("EQUIV index2:baz index2:foo index2:bar", "index2:baz"); + tester.assertSemantics("EQUIV index2:word index2:\"a phrase\"", "index2:word"); + tester.assertSemantics("EQUIV index2:\"a phrase\" index2:word", "index2:a index2:phrase"); + tester.assertSemantics("index2:other", "index2:other"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr index 9a147887207..32f8e86b59f 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/expansion.sr @@ -5,6 +5,4 @@ equiv1 +> =equiv2 =equiv3; testfield:[test] -> =testfield:e1 =testfield:e2 =testfield:e3; -synonymfield:[test] -> =[test]; - [test] :- foo, bar, baz; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr new file mode 100644 index 00000000000..4220f807733 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/synonyms.sr @@ -0,0 +1,11 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +index1:[synonyms1] -> =index1:[synonyms1*]; # Replace by equiv(foo, bar, baz) when the query contains foo, bar or baz +index1:[synonyms2] -> =index1:[synonyms2*]; # with phrase + + +index2:[synonyms1] +> =index2:[synonyms1*]; # Add equiv(foo, bar, baz) when the query contains foo, bar or baz +index2:[synonyms2] +> =index2:[synonyms2*]; # with phrase + +[synonyms1] :- foo, baz, bar; +[synonyms2] :- word, a phrase; diff --git a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java index 1e35b6447ff..fa8d237aee0 100644 --- a/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/cluster/test/ClusteredConnectionTestCase.java @@ -133,7 +133,7 @@ public class ClusteredConnectionTestCase { /** Represents a connection, e.g over http, in this test */ private static class Connection { - private String id; + private final String id; private boolean inService = true; diff --git a/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java b/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java index 054b752c067..6c73937f4fa 100644 --- a/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java @@ -2,6 +2,7 @@ package com.yahoo.search.query; import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.NullItem; import com.yahoo.prelude.query.WordItem; import org.junit.Assert; import org.junit.Test; @@ -41,4 +42,25 @@ public class QueryTreeTest { assertEquals("+(AND p1 p2) -n1.1 -n1.2 -n2.1 -n2.2", tree.toString()); } + @Test + public void getCorrectTreeSize() { + QueryTree nullTree = new QueryTree(new NullItem()); + assertEquals(0, nullTree.treeSize()); + + NotItem not1 = new NotItem(); + not1.addPositiveItem(new WordItem("p1")); + not1.addNegativeItem(new WordItem("n1.1")); + not1.addNegativeItem(new WordItem("n1.2")); + + NotItem not2 = new NotItem(); + not2.addPositiveItem(new WordItem("p2")); + not2.addNegativeItem(new WordItem("n2.1")); + not2.addNegativeItem(new WordItem("n2.2")); + + QueryTree tree = new QueryTree(not1); + tree.and(not2); + + assertEquals(8, tree.treeSize()); + } + } diff --git a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java index 9e56fba0e17..4f14ae066cb 100644 --- a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java @@ -63,8 +63,8 @@ public class SortingTestCase { private void requireThatChineseHasCorrectRules(Collator col) { final int reorderCodes [] = {UScript.HAN}; - assertEquals("8.0.0.0", col.getUCAVersion().toString()); - assertEquals("153.64.29.0", col.getVersion().toString()); + assertEquals("14.0.0.0", col.getUCAVersion().toString()); + assertEquals("153.112.40.0", col.getVersion().toString()); assertEquals(Arrays.toString(reorderCodes), Arrays.toString(col.getReorderCodes())); assertNotEquals("", ((RuleBasedCollator) col).getRules()); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java index a9792049fb7..511921ac047 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfileRegistryTest.java @@ -4,7 +4,10 @@ package com.yahoo.search.query.profile.compiled; import com.yahoo.search.query.profile.config.QueryProfilesConfig; import org.junit.Test; +import java.util.concurrent.Executors; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author gjoranv @@ -21,7 +24,9 @@ public class CompiledQueryProfileRegistryTest { .value("5"))) .build(); - var registry = new CompiledQueryProfileRegistry(config); + var executor = Executors.newCachedThreadPool(); + var registry = new CompiledQueryProfileRegistry(config, executor); + assertTrue(executor.shutdownNow().isEmpty()); var profile1 = registry.findQueryProfile("profile1"); assertEquals("5", profile1.get("hits")); } diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java index 60b6c6dbe84..3ed044601f0 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/DumpToolTestCase.java @@ -11,7 +11,7 @@ import static org.junit.Assert.assertTrue; */ public class DumpToolTestCase { - private String profileDir = "src/test/java/com/yahoo/search/query/profile/config/test/multiprofile"; + private final String profileDir = "src/test/java/com/yahoo/search/query/profile/config/test/multiprofile"; @Test public void testNoParameters() { @@ -25,12 +25,13 @@ public class DumpToolTestCase { @Test public void testNoDimensionValues() { - assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir).startsWith("a=general-a\n")); + System.out.println(new DumpTool().resolveAndDump("multiprofile1", profileDir)); + assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir).contains("a=general-a\n")); } @Test public void testAllParametersSet() { - assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir, "").startsWith("a=general-a\n")); + assertTrue(new DumpTool().resolveAndDump("multiprofile1", profileDir, "").contains("a=general-a\n")); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java index 4b8edfdde73..fa7508ea5ac 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java @@ -47,7 +47,6 @@ public class NameTestCase { assertLegalFieldName("a/b"); assertLegalFieldName("/a/b"); assertLegalFieldName("/a/b/"); - assertIllegalFieldName(""); assertIllegalFieldName("aBc.dooEee.ce_d.-some-other.moreHere", "Could not set 'aBc.dooEee.ce_d.-some-other.moreHere' to 'anyValue'", "Illegal name '-some-other'"); diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index f3a71af0b9e..637666a6a19 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -54,6 +54,7 @@ import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; +import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.trace.TraceNode; import org.junit.Before; import org.junit.Test; @@ -69,6 +70,7 @@ import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Functional testing of {@link JsonRenderer}. @@ -165,11 +167,26 @@ public class JsonRendererTestCase { h.setField("tensor_mixed", new TensorFieldValue(Tensor.from("tensor(x{},y[2]):{a:[1,2], b:[3,4]}"))); h.setField("summaryfeatures", summaryFeatures); - Result r = new Result(new Query("/?format.tensors=short")); - r.hits().add(h); - r.setTotalHitCount(1L); - String summary = render(r); - assertEqualJson(expected, summary); + Result result1 = new Result(new Query("/?presentation.format.tensors=short")); + result1.hits().add(h); + result1.setTotalHitCount(1L); + String summary1 = render(result1); + assertEqualJson(expected, summary1); + + Result result2 = new Result(new Query("/?format.tensors=short")); + result2.hits().add(h); + result2.setTotalHitCount(1L); + String summary2 = render(result2); + assertEqualJson(expected, summary2); + + try { + render(new Result(new Query("/?presentation.format.tensors=unknown"))); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("Could not set 'presentation.format.tensors' to 'unknown': Value must be 'long' or 'short', not 'unknown'", + Exceptions.toMessageString(e)); + } } @Test @@ -1276,7 +1293,9 @@ public class JsonRendererTestCase { " f1: [ 'v1', { mykey1: 'myvalue1', mykey2: 'myvalue2' } ]," + " f2: { i1: 'v2', i2: { mykey3: 'myvalue3' }, i3: 'v3' }," + " f3: { j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }," + - " f4: { mykey4: 'myvalue4', mykey5: 'myvalue5' }" + + " f4: { mykey4: 'myvalue4', mykey5: 'myvalue5' }," + + " f5: { '10001': 'myvalue6', '10002': 'myvalue7' }," + + " f6: { i4: 'v6', i5: { '-17': 'myvalue8', '-42': 'myvalue9' } }" + " }" + " } ]" + "}}"); @@ -1285,6 +1304,8 @@ public class JsonRendererTestCase { h.setField("f2", dataFromSimplified("{ i1: 'v2', i2: [ { key: 'mykey3', value: 'myvalue3' } ], i3: 'v3' }")); h.setField("f3", dataFromSimplified("{ j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }")); h.setField("f4", dataFromSimplified("[ { key: 'mykey4', value: 'myvalue4' }, { key: 'mykey5', value: 'myvalue5' } ]")); + h.setField("f5", dataFromSimplified("[ { key: 10001, value: 'myvalue6' }, { key: 10002, value: 'myvalue7' } ]")); + h.setField("f6", dataFromSimplified("{ i4: 'v6', i5: [ {key: -17, value: 'myvalue8' }, { key: -42, value: 'myvalue9' } ] }")); r.hits().add(h); r.setTotalHitCount(1L); String summary = render(r); @@ -1298,7 +1319,51 @@ public class JsonRendererTestCase { " f1: [ 'v1', [ { key: 'mykey1', value: 'myvalue1' }, { key: 'mykey2', value: 'myvalue2' } ] ]," + " f2: { i1: 'v2', i2: [ { key: 'mykey3', value: 'myvalue3' } ], i3: 'v3' }," + " f3: { j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }," + - " f4: { mykey4: 'myvalue4', mykey5: 'myvalue5' }" + + " f4: { mykey4: 'myvalue4', mykey5: 'myvalue5' }," + + " f5: [ { key: 10001, value: 'myvalue6' }, { key: 10002, value: 'myvalue7' } ]," + + " f6: { i4: 'v6', i5: [ { key: -17, value: 'myvalue8' }, { key: -42, value: 'myvalue9' } ] }" + + " }" + + " } ]" + + "}}"); + r.hits().add(h); + r.setTotalHitCount(1L); + summary = render(r); + assertEqualJson(expected.toString(), summary); + } + + @Test + public void testWsetInFields() throws IOException, InterruptedException, ExecutionException { + Result r = new Result(new Query("/?renderer.json.jsonWsets=true")); + var expected = dataFromSimplified( + "{root: { id:'toplevel', relevance:1.0, fields: { totalCount: 1 }," + + " children: [ { id: 'myHitName', relevance: 1.0," + + " fields: { " + + " f1: [ 'v1', { mykey1: 10, mykey2: 20 } ]," + + " f2: { i1: 'v2', i2: { mykey3: 30 }, i3: 'v3' }," + + " f3: { j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }," + + " f4: { mykey4: 40, mykey5: 50 }" + + " }" + + " } ]" + + "}}"); + Hit h = new Hit("myHitName"); + h.setField("f1", dataFromSimplified("[ 'v1', [ { item: 'mykey1', weight: 10 }, { item: 'mykey2', weight: 20 } ] ]")); + h.setField("f2", dataFromSimplified("{ i1: 'v2', i2: [ { item: 'mykey3', weight: 30 } ], i3: 'v3' }")); + h.setField("f3", dataFromSimplified("{ j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }")); + h.setField("f4", dataFromSimplified("[ { item: 'mykey4', weight: 40 }, { item: 'mykey5', weight: 50 } ]")); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected.toString(), summary); + + r = new Result(new Query("/?renderer.json.jsonWsets=false")); + expected = dataFromSimplified( + "{root:{id:'toplevel',relevance:1.0,fields:{totalCount:1}," + + " children: [ { id: 'myHitName', relevance: 1.0," + + " fields: { " + + " f1: [ 'v1', [ { item: 'mykey1', weight: 10 }, { item: 'mykey2', weight: 20 } ] ]," + + " f2: { i1: 'v2', i2: [ { item: 'mykey3', weight: 30 } ], i3: 'v3' }," + + " f3: { j1: 42, j2: 17.75, j3: [ 'v4', 'v5' ] }," + + " f4: [ { item: 'mykey4', weight: 40 }, { item: 'mykey5', weight: 50 } ]" + " }" + " } ]" + "}}"); diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 14fdd047391..9a760bfb0cb 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.search.searchers; import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.config.subscription.FileSource; import com.yahoo.config.subscription.RawSource; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; @@ -18,7 +19,11 @@ import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; import org.junit.Test; + +import java.io.File; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -32,8 +37,8 @@ public class ValidateNearestNeighborTestCase { public ValidateNearestNeighborTestCase() { searcher = new ValidateNearestNeighborSearcher( ConfigGetter.getConfig(AttributesConfig.class, - "raw:", - new RawSource("attribute[5]\n" + + "raw:" + + "attribute[5]\n" + "attribute[0].name simple\n" + "attribute[0].datatype INT32\n" + "attribute[1].name dvector\n" + @@ -57,7 +62,7 @@ public class ValidateNearestNeighborTestCase { "attribute[7].name threetypes\n" + "attribute[7].datatype TENSOR\n" + "attribute[7].tensortype tensor(x{})\n" - ))); + )); } private static TensorType tt_dense_dvector_42 = TensorType.fromSpec("tensor(x[42])"); diff --git a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java index 86ffb1f9830..01e360858c1 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/test/ValidateMatchPhaseSearcherTestCase.java @@ -27,8 +27,8 @@ public class ValidateMatchPhaseSearcherTestCase { public ValidateMatchPhaseSearcherTestCase() { searcher = new ValidateMatchPhaseSearcher( ConfigGetter.getConfig(AttributesConfig.class, - "raw:", - new RawSource("attribute[4]\n" + + "raw:" + + "attribute[4]\n" + "attribute[0].name ok\n" + "attribute[0].datatype INT32\n" + "attribute[0].collectiontype SINGLE\n" + @@ -45,7 +45,7 @@ public class ValidateMatchPhaseSearcherTestCase { "attribute[3].datatype INT32\n" + "attribute[3].collectiontype ARRAY\n" + "attribute[3].fastsearch true" - ))); + )); } private static String getErrorMatch(String attribute) { diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index f6273fdf723..a4fd37145c8 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.search.test; import com.yahoo.component.chain.Chain; +import com.yahoo.data.JsonProducer; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detection; @@ -15,6 +16,7 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; +import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; @@ -36,10 +38,12 @@ import com.yahoo.search.query.profile.DimensionValues; import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.yolean.Exceptions; +import org.json.JSONObject; import org.junit.Ignore; import org.junit.Test; @@ -69,6 +73,17 @@ import static org.junit.Assert.fail; public class QueryTestCase { @Test + public void testIt() throws Exception { + JSONObject newroot = new JSONObject("{\"key\": 3}"); + var hit = new FastHit(); + hit.setField("data", (JsonProducer)s -> s.append(newroot)); + var field = hit.getField("data"); + if (field instanceof JsonProducer) { + System.out.println((((JsonProducer) field).toJson())); + } + } + + @Test public void testSimpleFunctionality() { Query q = new Query(QueryTestCase.httpEncode("/sdfsd.html?query=this is a simple query&aParameter")); assertEquals("this is a simple query", q.getModel().getQueryString()); @@ -1078,6 +1093,49 @@ public class QueryTestCase { query.getSelect().getGrouping().toString()); } + /** + * Tests that the value presentation.format.tensors can be set in a query profile. + * This is special because presentation.format is a native query profile. + */ + @Test + public void testSettingNativeQueryProfileValueInQueryProfile() { + { + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("default"); + profile.set("presentation.format.tensors", "short", Map.of(), registry); + registry.register(profile); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + Query query = new Query("?query=foo", cRegistry.findQueryProfile("default")); + assertTrue(query.getPresentation().getTensorShortForm()); + } + + { // Same as above but also set presentation.format + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfile profile = new QueryProfile("default"); + profile.set("presentation.format", "xml", Map.of(), registry); + profile.set("presentation.format.tensors", "short", Map.of(), registry); + registry.register(profile); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + Query query = new Query("?query=foo", cRegistry.findQueryProfile("default")); + assertEquals("xml", query.getPresentation().getFormat()); + assertTrue(query.getPresentation().getTensorShortForm()); + } + + { // Set presentation.format with a typed query profile type + QueryProfileRegistry registry = new QueryProfileRegistry(); + QueryProfileType type = new QueryProfileType("mytype"); + type.inherited().add(registry.getType("native")); + registry.getTypeRegistry().register(type); + type.addField(new FieldDescription("ranking.features.query(embedding)", "tensor(x[5])"), + registry.getTypeRegistry()); + QueryProfile profile = new QueryProfile("default"); + profile.setType(type); + registry.register(profile); + CompiledQueryProfileRegistry cRegistry = registry.compile(); + Query query = new Query("?query=foo&presentation.format=xml", cRegistry.findQueryProfile("default")); + } + } + private void assertDetectionText(String expectedDetectionText, String queryString, String ... indexSpecs) { Query q = new Query(httpEncode("/?query=" + queryString)); SearchDefinition sd = new SearchDefinition("testSearchDefinition"); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 5d3a95efc78..0385de5bebf 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -24,7 +24,9 @@ import com.yahoo.search.Searcher; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; -import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS;; +import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS; +import static com.yahoo.prelude.fastsearch.VespaBackEndSearcher.SORTABLE_ATTRIBUTES_SUMMARY_CLASS; + /** * Test translation of fields and sources in YQL to the associated concepts in Vespa. @@ -52,7 +54,7 @@ public class YqlFieldAndSourceTestCase { mockBackend.addResult(query, result); mockBackend.addSummaryClassByCopy(DEFAULT_SUMMARY_CLASS, Arrays.asList(FIELD1, FIELD2)); - mockBackend.addSummaryClassByCopy(Execution.ATTRIBUTEPREFETCH, Arrays.asList(FIELD2)); + mockBackend.addSummaryClassByCopy(SORTABLE_ATTRIBUTES_SUMMARY_CLASS, Arrays.asList(FIELD2)); mockBackend.addSummaryClassByCopy(THIRD_OPTION, Arrays.asList(FIELD3)); DocumentdbInfoConfig config = new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder() @@ -90,7 +92,7 @@ public class YqlFieldAndSourceTestCase { new Fields.Builder().name(FIELD2).type("string"))), new Summaryclass.Builder() .id(1) - .name(Execution.ATTRIBUTEPREFETCH) + .name(SORTABLE_ATTRIBUTES_SUMMARY_CLASS) .fields(Arrays.asList(new Fields.Builder().name(FIELD2).type("string"))), new Summaryclass.Builder() .id(2) @@ -112,7 +114,7 @@ public class YqlFieldAndSourceTestCase { execution.fill(result); assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertFalse(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } @Test @@ -123,7 +125,7 @@ public class YqlFieldAndSourceTestCase { assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertTrue(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertTrue(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } @Test @@ -134,7 +136,7 @@ public class YqlFieldAndSourceTestCase { assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertFalse(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } @Test @@ -145,7 +147,7 @@ public class YqlFieldAndSourceTestCase { assertEquals(1, result.getConcreteHitCount()); assertTrue(result.hits().get(0).isFilled(THIRD_OPTION)); assertTrue(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); - assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); + assertFalse(result.hits().get(0).isFilled(SORTABLE_ATTRIBUTES_SUMMARY_CLASS)); } } diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java index cc70f46fca1..1d07cafeda9 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsVisitorTestCase.java @@ -243,7 +243,6 @@ public class VdsVisitorTestCase { assertEquals(DocumentProtocol.Priority.VERY_HIGH, params.getPriority()); } } - assertEquals(-1, params.getMaxFirstPassHits()); if (qa.maxBucketsPerVisitor != 0) { assertEquals(qa.maxBucketsPerVisitor, params.getMaxBucketsPerVisitor()); } else { |