diff options
author | Arnstein Ressem <aressem@oath.com> | 2018-08-09 10:27:43 +0200 |
---|---|---|
committer | Arnstein Ressem <aressem@oath.com> | 2018-08-09 10:27:43 +0200 |
commit | 6d61753ac389a884430be9e2eb9bbbd216ea4db5 (patch) | |
tree | 9a1b00c5ac40a0c0ea9d822658e225c90adab84b /container-search/src/main | |
parent | b80a8292c21e0c0dd678024928077e3e268de789 (diff) | |
parent | e2887cb7299438c02bc49d888aaaf2e51631ace9 (diff) |
Merge branch 'master' into aressem/kill-mbuild
Diffstat (limited to 'container-search/src/main')
15 files changed, 1389 insertions, 27 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java index 33e7ad4bccc..dd5b3caf0c5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/CacheControl.java @@ -18,7 +18,7 @@ import java.util.Optional; */ public class CacheControl { - private static final CompoundName nocachewrite=new CompoundName("nocachewrite"); + public static final CompoundName nocachewrite=new CompoundName("nocachewrite"); /** Whether this CacheControl actually should cache hits at all. */ private final boolean activeCache; diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 3a2e922f9e0..0d25e71dd76 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -125,13 +125,15 @@ public class FastHit extends Hit { * found. The row count is used to decode the part id into a column and a * row number: the number of n least significant bits required to hold the * highest row number are the row bits, the rest are column bits. + * + * Note: Remove partId when all dispatching happens from the container dispatcher, not fdispatch */ public void setPartId(int partId) { this.partId = partId; } /** Returns the index of the node this hit originated at */ public int getDistributionKey() { return distributionKey; } - /** Returns the index of the node this hit originated at */ + /** Sets the index of the node this hit originated at */ public void setDistributionKey(int distributionKey) { this.distributionKey = distributionKey; } /** diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java index 314ca38f092..060fd598245 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java @@ -35,7 +35,7 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; @Before({STEMMING, ACCENT_REMOVAL}) public class RecallSearcher extends Searcher { - private static final CompoundName recallName=new CompoundName("recall"); + public static final CompoundName recallName=new CompoundName("recall"); @Override public Result search(Query query, Execution execution) { diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index dd3cc853742..cfb1c9a26be 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -20,6 +20,7 @@ import com.yahoo.search.query.Presentation; import com.yahoo.search.query.Properties; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.Ranking; +import com.yahoo.search.query.Select; import com.yahoo.search.query.SessionId; import com.yahoo.search.query.Sorting; import com.yahoo.search.query.Sorting.AttributeSorter; @@ -100,7 +101,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { ADVANCED(3,"adv"), WEB(4,"web"), PROGRAMMATIC(5, "prog"), - YQL(6, "yql"); + YQL(6, "yql"), + SELECT(7, "select");; private final int intValue; private final String stringValue; @@ -170,6 +172,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** How results of this query should be presented */ private Presentation presentation = new Presentation(this); + /** The selection of where-clause and grouping */ + private Select select = new Select(this); + //---------------- Tracing ---------------------------------------------------- private static Logger log = Logger.getLogger(Query.class.getName()); @@ -188,6 +193,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { public static final CompoundName GROUPING_SESSION_CACHE = new CompoundName("groupingSessionCache"); public static final CompoundName TIMEOUT = new CompoundName("timeout"); + private static QueryProfileType argumentType; static { argumentType = new QueryProfileType("native"); @@ -206,10 +212,12 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { argumentType.addField(new FieldDescription(Presentation.PRESENTATION, new QueryProfileFieldType(Presentation.getArgumentType()))); argumentType.addField(new FieldDescription(Ranking.RANKING, new QueryProfileFieldType(Ranking.getArgumentType()))); argumentType.addField(new FieldDescription(Model.MODEL, new QueryProfileFieldType(Model.getArgumentType()))); + argumentType.addField(new FieldDescription(Select.SELECT, new QueryProfileFieldType(Select.getArgumentType()))); argumentType.freeze(); } public static QueryProfileType getArgumentType() { return argumentType; } + /** The aliases of query properties */ private static Map<String,CompoundName> propertyAliases; static { @@ -218,6 +226,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { addAliases(Ranking.getArgumentType(), propertyAliasesBuilder); addAliases(Model.getArgumentType(), propertyAliasesBuilder); addAliases(Presentation.getArgumentType(), propertyAliasesBuilder); + addAliases(Select.getArgumentType(), propertyAliasesBuilder); propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder); } private static void addAliases(QueryProfileType arguments, Map<String, CompoundName> aliases) { @@ -237,6 +246,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { registry.register(Query.getArgumentType().unfrozen()); registry.register(Ranking.getArgumentType().unfrozen()); registry.register(Model.getArgumentType().unfrozen()); + registry.register(Select.getArgumentType().unfrozen()); registry.register(Presentation.getArgumentType().unfrozen()); registry.register(DefaultProperties.argumentType.unfrozen()); } @@ -381,6 +391,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { setFrom(properties,Model.getArgumentType(), context); setFrom(properties,Presentation.getArgumentType(), context); setFrom(properties,Ranking.getArgumentType(), context); + setFrom(properties, Select.getArgumentType(), context); } /** @@ -511,11 +522,15 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { * Get the appropriate timeout for the query. * * @return timeout in milliseconds - **/ + */ public long getTimeLeft() { return getTimeout() - getDurationTime(); } + /** + * @deprecated do not use + */ + @Deprecated // TODO: Remove on Vespa 7 public boolean requestHasProperty(String name) { return httpRequest.hasProperty(name); } @@ -976,6 +991,9 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns the presentation to be used for this query, never null */ public Presentation getPresentation() { return presentation; } + /** Returns the select to be used for this query, never null */ + public Select getSelect() { return select; } + /** Returns the ranking to be used for this query, never null */ public Ranking getRanking() { return ranking; } diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java index b59578ab6a3..fdbee9c8f11 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java @@ -140,6 +140,8 @@ public class VespaSearcher extends ConfiguredHTTPProviderSearcher { return Query.Type.PROGRAMMATIC; } else if (providerQueryType == ProviderConfig.QueryType.YQL) { return Query.Type.YQL; + } else if (providerQueryType == ProviderConfig.QueryType.SELECT) { + return Query.Type.SELECT; } else { throw new RuntimeException("Query type " + providerQueryType + " unsupported."); } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 7406d492bc8..5ac1f834031 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -283,13 +283,16 @@ public class SearchHandler extends LoggingRequestHandler { private HttpSearchResponse handleBody(HttpRequest request){ - // Find query profile - String queryProfileName = request.getProperty("queryProfile"); + + Map<String, String> requestMap = requestMapFromRequest(request); + + // Get query profile + String queryProfileName = requestMap.getOrDefault("queryProfile", null); CompiledQueryProfile queryProfile = queryProfileRegistry.findQueryProfile(queryProfileName); - boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request); - Query query = queryFromRequest(request, queryProfile); + Query query = new Query(request, requestMap, queryProfile); + boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request); boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarkOutput, request.getJDiscRequest().headers()); // Find and execute search chain if we have a valid query @@ -558,9 +561,10 @@ public class SearchHandler extends LoggingRequestHandler { return searchChainRegistry; } + private Map<String, String> requestMapFromRequest(HttpRequest request) { - private Query queryFromRequest(HttpRequest request, CompiledQueryProfile queryProfile){ - if (request.getMethod() == com.yahoo.jdisc.http.HttpRequest.Method.POST && request.getHeader(com.yahoo.jdisc.http.HttpHeaders.Names.CONTENT_TYPE).equals(JSON_CONTENT_TYPE)) { + if (request.getMethod() == com.yahoo.jdisc.http.HttpRequest.Method.POST + && JSON_CONTENT_TYPE.equals(request.getHeader(com.yahoo.jdisc.http.HttpHeaders.Names.CONTENT_TYPE))) { Inspector inspector; try { byte[] byteArray = IOUtils.readBytes(request.getData(), 1 << 20); @@ -576,16 +580,26 @@ public class SearchHandler extends LoggingRequestHandler { // Create request-mapping Map<String, String> requestMap = new HashMap<>(); createRequestMapping(inspector, requestMap, ""); - return new Query(request, requestMap, queryProfile); + // Throws QueryException if query contains both yql- and select-parameter + if (requestMap.containsKey("yql") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) { + throw new QueryException("Illegal query: Query contains both yql- and select-parameter"); + } + + // Throws QueryException if query contains both query- and select-parameter + if (requestMap.containsKey("query") && (requestMap.containsKey("select.where") || requestMap.containsKey("select.grouping")) ) { + throw new QueryException("Illegal query: Query contains both query- and select-parameter"); + } + + return requestMap; } else { - return new Query(request, queryProfile); + return request.propertyMap(); } } - public void createRequestMapping(Inspector inspector, Map<String, String> map, String parent){ + public void createRequestMapping(Inspector inspector, Map<String, String> map, String parent) { inspector.traverse((ObjectTraverser) (key, value) -> { String qualifiedKey = parent + key; switch (value.type()) { @@ -605,6 +619,10 @@ public class SearchHandler extends LoggingRequestHandler { map.put(qualifiedKey, value.asString()); break; case OBJECT: + if (qualifiedKey.equals("select.where") || qualifiedKey.equals("select.grouping")){ + map.put(qualifiedKey, value.toString()); + break; + } createRequestMapping(value, map, qualifiedKey+"."); break; } @@ -612,8 +630,6 @@ public class SearchHandler extends LoggingRequestHandler { }); } - - } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index dc7a61344cb..bd0c229085b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -78,11 +78,11 @@ public class Model implements Cloneable { private String defaultIndex = null; private Query.Type type = Query.Type.ALL; private Query parent; - private Set<String> sources=new LinkedHashSet<>(); - private Set<String> restrict=new LinkedHashSet<>(); + private Set<String> sources = new LinkedHashSet<>(); + private Set<String> restrict = new LinkedHashSet<>(); private String searchPath; private String documentDbName = null; - private Execution execution=new Execution(new Execution.Context(null, null, null, null, null)); + private Execution execution = new Execution(new Execution.Context(null, null, null, null, null)); public Model(Query query) { setParent(query); @@ -101,7 +101,7 @@ public class Model implements Cloneable { */ @Deprecated public void traceLanguage() { - if (getParent().getTraceLevel()<2) return; + if (getParent().getTraceLevel() < 2) return; if (language != null) { getParent().trace("Language " + getLanguage() + " specified directly as a parameter", false, 2); } @@ -235,9 +235,9 @@ public class Model implements Cloneable { * Passing null causes this to be set to an empty string. */ public void setQueryString(String queryString) { - if (queryString==null) queryString=""; + if (queryString == null) queryString=""; this.queryString = queryString; - queryTree=null; // Cause parsing of the new query string next time the tree is accessed + queryTree = null; // Cause parsing of the new query string next time the tree is accessed } /** @@ -249,7 +249,7 @@ public class Model implements Cloneable { public String getQueryString() { return queryString; } /** - * Returns the query as an object structure. + * Returns the query as an object structure. Remember to have the correct Query.Type set. * This causes parsing of the query string if it has changed since this was last called * (i.e query parsing is lazy) */ diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java new file mode 100644 index 00000000000..3ffc6bddb24 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -0,0 +1,110 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + +import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.Query; +import com.yahoo.search.grouping.GroupingRequest; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.query.parser.ParserFactory; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.yql.VespaGroupingStep; + + + +/** + * The parameters defining the where-clause and groping of a query + * + * @author henrhoi + */ +public class Select implements Cloneable { + + /** The type representing the property arguments consumed by this */ + private static final QueryProfileType argumentType; + private static final CompoundName argumentTypeName; + + public static final String SELECT = "select"; + public static final String WHERE = "where"; + public static final String GROUPING = "grouping"; + + + private static Model model; + private Query parent; + private String where = ""; + private String grouping = ""; + + static { + argumentType = new QueryProfileType(SELECT); + argumentType.setStrict(true); + argumentType.setBuiltin(true); + argumentType.addField(new FieldDescription(WHERE, "string", "where")); + argumentType.addField(new FieldDescription(GROUPING, "string", "grouping")); + argumentType.freeze(); + argumentTypeName=new CompoundName(argumentType.getId().getName()); + } + + public static QueryProfileType getArgumentType() { return argumentType; } + + public Select(String where, String grouping){ + this.where = where; + this.grouping = grouping; + } + + public Select(Query query) { + setParent(query); + model = query.getModel(); + } + + + /** Returns the query owning this, never null */ + private Query getParent() { return parent; } + + + /** Assigns the query owning this */ + public void setParent(Query parent) { + if (parent==null) throw new NullPointerException("A query models owner cannot be null"); + this.parent = parent; + } + + + /** Set the where-clause for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */ + public void setWhere(String where) { + this.where = where; + model.setType(SELECT); + + // Setting the queryTree to null + model.setQueryString(null); + } + + + /** Returns the where-clause in the query */ + public String getWhereString(){ + return this.where; + } + + + /** Set the grouping-string for the query. Must be a JSON-string, with the format described in the Select Reference doc - https://docs.vespa.ai/documentation/reference/select-reference.html. */ + public void setGrouping(String grouping){ + this.grouping = grouping; + SelectParser parser = (SelectParser) ParserFactory.newInstance(Query.Type.SELECT, new ParserEnvironment()); + + for (VespaGroupingStep step : parser.getGroupingSteps(grouping)) { + GroupingRequest.newInstance(parent) + .setRootOperation(step.getOperation()) + .continuations().addAll(step.continuations()); + } + } + + + /** Returns the grouping in the query */ + public String getGroupingString(){ + return this.grouping; + } + + + @Override + public String toString() { + return "where: [" + where + "], grouping: [" + grouping+ "]"; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java new file mode 100644 index 00000000000..13ebacb62ef --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -0,0 +1,1185 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + + +import com.google.common.base.Preconditions; +import com.yahoo.collections.LazyMap; +import com.yahoo.language.Language; +import com.yahoo.language.process.Normalizer; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.DotProductItem; +import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.ExactStringItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.ONearItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.QueryException; +import com.yahoo.prelude.query.RangeItem; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SameElementItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.prelude.query.WandItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; +import com.yahoo.search.grouping.request.GroupingOperation; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.Parser; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.yql.VespaGroupingStep; +import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.vespa.config.SlimeUtils; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.slime.Type.ARRAY; +import static com.yahoo.slime.Type.DOUBLE; +import static com.yahoo.slime.Type.LONG; +import static com.yahoo.slime.Type.OBJECT; +import static com.yahoo.slime.Type.STRING; + +/** + * The Select query language. + * + * This class will be parsing the Select parameters, and will be used when the query has the SELECT-type. + * + * @author henrhoi + */ + + +public class SelectParser implements Parser { + + Parsable query; + private final IndexFacts indexFacts; + private final Map<Integer, TaggableItem> identifiedItems = LazyMap.newHashMap(); + private final List<ConnectedItem> connectedItems = new ArrayList<>(); + private final Normalizer normalizer; + private final ParserEnvironment environment; + private IndexFacts.Session indexFactsSession; + + + + /** YQL parameters and functions */ + + private static final String DESCENDING_HITS_ORDER = "descending"; + private static final String ASCENDING_HITS_ORDER = "ascending"; + private static final Integer DEFAULT_TARGET_NUM_HITS = 10; + private static final String ORIGIN_LENGTH = "length"; + private static final String ORIGIN_OFFSET = "offset"; + private static final String ORIGIN = "origin"; + private static final String ORIGIN_ORIGINAL = "original"; + private static final String CONNECTION_ID = "id"; + private static final String CONNECTION_WEIGHT = "weight"; + private static final String CONNECTIVITY = "connectivity"; + private static final String ANNOTATIONS = "annotations"; + private static final String NFKC = "nfkc"; + private static final String USER_INPUT_LANGUAGE = "language"; + private static final String ACCENT_DROP = "accentDrop"; + private static final String ALTERNATIVES = "alternatives"; + private static final String AND_SEGMENTING = "andSegmenting"; + private static final String DISTANCE = "distance"; + private static final String DOT_PRODUCT = "dotProduct"; + private static final String EQUIV = "equiv"; + private static final String FILTER = "filter"; + private static final String HIT_LIMIT = "hitLimit"; + private static final String IMPLICIT_TRANSFORMS = "implicitTransforms"; + private static final String LABEL = "label"; + private static final String NEAR = "near"; + private static final String NORMALIZE_CASE = "normalizeCase"; + private static final String ONEAR = "onear"; + private static final String PHRASE = "phrase"; + private static final String PREDICATE = "predicate"; + private static final String PREFIX = "prefix"; + private static final String RANKED = "ranked"; + private static final String RANK = "rank"; + private static final String SAME_ELEMENT = "sameElement"; + private static final String SCORE_THRESHOLD = "scoreThreshold"; + private static final String SIGNIFICANCE = "significance"; + private static final String STEM = "stem"; + private static final String SUBSTRING = "substring"; + private static final String SUFFIX = "suffix"; + private static final String TARGET_NUM_HITS = "targetNumHits"; + private static final String THRESHOLD_BOOST_FACTOR = "thresholdBoostFactor"; + private static final String UNIQUE_ID = "id"; + private static final String USE_POSITION_DATA = "usePositionData"; + private static final String WAND = "wand"; + private static final String WEAK_AND = "weakAnd"; + private static final String WEIGHTED_SET = "weightedSet"; + private static final String WEIGHT = "weight"; + private static final String AND = "and"; + private static final String AND_NOT = "and_not"; + private static final String OR = "or"; + private static final String EQ = "equals"; + private static final String RANGE = "range"; + private static final String CONTAINS = "contains"; + private static final String MATCHES = "matches"; + private static final String CALL = "call"; + private static final List<String> FUNCTION_CALLS = Arrays.asList(WAND, WEIGHTED_SET, DOT_PRODUCT, PREDICATE, RANK, WEAK_AND); + + /**************************************/ + + + + public SelectParser(ParserEnvironment environment) { + indexFacts = environment.getIndexFacts(); + normalizer = environment.getLinguistics().getNormalizer(); + + this.environment = environment; + } + + + @Override + public QueryTree parse(Parsable query) { + indexFactsSession = indexFacts.newSession(query.getSources(), query.getRestrict()); + connectedItems.clear(); + identifiedItems.clear(); + this.query = query; + + return buildTree(); + } + + + + private QueryTree buildTree() { + Inspector inspector = SlimeUtils.jsonToSlime(this.query.getSelect().getWhereString().getBytes()).get(); + if (inspector.field("error_message").valid()){ + throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8)); + } + + Item root = walkJson(inspector); + connectItems(); + QueryTree newTree = new QueryTree(root); + + return newTree; + } + + + private Item walkJson(Inspector inspector){ + final Item[] item = {null}; + inspector.traverse((ObjectTraverser) (key, value) -> { + String type = (FUNCTION_CALLS.contains(key)) ? CALL : key; + + switch (type) { + + case AND: + item[0] = buildAnd(key, value); + break; + case AND_NOT: + item[0] = buildNotAnd(key, value); + break; + case OR: + item[0] = buildOr(key, value); + break; + case EQ: + item[0] = buildEquals(key, value); + break; + case RANGE: + item[0] = buildRange(key, value); + break; + case CONTAINS: + item[0] = buildTermSearch(key, value); + break; + case MATCHES: + item[0] = buildRegExpSearch(key, value); + break; + case CALL: + item[0] = buildFunctionCall(key, value); + break; + default: + throw newUnexpectedArgumentException(key, AND, CALL, CONTAINS, EQ, OR, RANGE, AND_NOT); + } + }); + return item[0]; + } + + + public List<VespaGroupingStep> getGroupingSteps(String grouping){ + List<VespaGroupingStep> groupingSteps = new ArrayList<>(); + List<String> groupingOperations = getOperations(grouping); + for (String groupingString : groupingOperations){ + GroupingOperation groupingOperation = GroupingOperation.fromString(groupingString); + VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); + groupingSteps.add(groupingStep); + } + return groupingSteps; + } + + private List<String> getOperations(String grouping) { + List<String> operations = new ArrayList<>(); + Inspector inspector = SlimeUtils.jsonToSlime(grouping.getBytes()).get(); + if (inspector.field("error_message").valid()){ + throw new QueryException("Illegal query: "+inspector.field("error_message").asString() + ", at: "+ new String(inspector.field("offending_input").asData(), StandardCharsets.UTF_8)); + } + + inspector.traverse( (ArrayTraverser) (key, value) -> { + String groupingString = value.toString(); + groupingString = groupingString.replace(" ", "").replace("\"", "").replace("\'", "").replace(":{", "(").replace(":", "(").replace("}", ")").replace(",", ")"); + groupingString = groupingString.substring(1, groupingString.length()); + operations.add(groupingString); + }); + + return operations; + + } + + + @NonNull + private Item buildFunctionCall(String key, Inspector value) { + switch (key) { + case WAND: + return buildWand(key, value); + case WEIGHTED_SET: + return buildWeightedSet(key, value); + case DOT_PRODUCT: + return buildDotProduct(key, value); + case PREDICATE: + return buildPredicate(key, value); + case RANK: + return buildRank(key, value); + case WEAK_AND: + return buildWeakAnd(key, value); + default: + throw newUnexpectedArgumentException(key, DOT_PRODUCT, RANK, WAND, WEAK_AND, WEIGHTED_SET, PREDICATE); + } + } + + + private void addItemsFromInspector(CompositeItem item, Inspector inspector){ + if (inspector.type() == ARRAY){ + inspector.traverse((ArrayTraverser) (index, new_value) -> { + item.addItem(walkJson(new_value)); + }); + + } else if (inspector.type() == OBJECT){ + if (inspector.field("children").valid()){ + inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { + item.addItem(walkJson(new_value)); + }); + } + + } + } + + + private Inspector getChildren(Inspector inspector){ + if (inspector.type() == ARRAY){ + return inspector; + + } else if (inspector.type() == OBJECT){ + if (inspector.field("children").valid()){ + return inspector.field("children"); + } + if (inspector.field(1).valid()){ + return inspector.field(1); + } + } + return null; + } + + + private HashMap<Integer, Inspector> getChildrenMap(Inspector inspector){ + HashMap<Integer, Inspector> children = new HashMap<>(); + if (inspector.type() == ARRAY){ + inspector.traverse((ArrayTraverser) (index, new_value) -> { + children.put(index, new_value); + }); + + } else if (inspector.type() == OBJECT){ + if (inspector.field("children").valid()){ + inspector.field("children").traverse((ArrayTraverser) (index, new_value) -> { + children.put(index, new_value); + }); + } + } + return children; + } + + + private Inspector getAnnotations(Inspector inspector){ + if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ + return inspector.field("attributes"); + } + return null; + } + + + private HashMap<String, Inspector> getAnnotationMapFromAnnotationInspector(Inspector annotation){ + HashMap<String, Inspector> attributes = new HashMap<>(); + if (annotation.type() == OBJECT){ + annotation.traverse((ObjectTraverser) (index, new_value) -> { + attributes.put(index, new_value); + }); + } + return attributes; + } + + + private HashMap<String, Inspector> getAnnotationMap(Inspector inspector){ + HashMap<String, Inspector> attributes = new HashMap<>(); + if (inspector.type() == OBJECT && inspector.field("attributes").valid()){ + inspector.field("attributes").traverse((ObjectTraverser) (index, new_value) -> { + attributes.put(index, new_value); + }); + } + return attributes; + } + + + private <T> T getAnnotation(String annotationName, HashMap<String, Inspector> annotations, Class<T> expectedClass, T defaultValue) { + return (annotations.get(annotationName) == null) ? defaultValue : expectedClass.cast(annotations.get(annotationName).asString()); + } + + + private Boolean getBoolAnnotation(String annotationName, HashMap<String, Inspector> annotations, Boolean defaultValue) { + if (annotations != null){ + Inspector annotation = annotations.getOrDefault(annotationName, null); + if (annotation != null){ + return annotation.asBool(); + } + } + return defaultValue; + } + + + private Integer getIntegerAnnotation(String annotationName, HashMap<String, Inspector> annotations, Integer defaultValue) { + if (annotations != null){ + Inspector annotation = annotations.getOrDefault(annotationName, null); + if (annotation != null){ + return (int)annotation.asLong(); + } + } + return defaultValue; + } + + + private Double getDoubleAnnotation(String annotationName, HashMap<String, Inspector> annotations, Double defaultValue) { + if (annotations != null){ + Inspector annotation = annotations.getOrDefault(annotationName, null); + if (annotation != null){ + return annotation.asDouble(); + } + } + return defaultValue; + } + + + private Inspector getAnnotationAsInspectorOrNull(String annotationName, HashMap<String, Inspector> annotations) { + return annotations.get(annotationName); + } + + + @NonNull + private CompositeItem buildAnd(String key, Inspector value) { + AndItem andItem = new AndItem(); + addItemsFromInspector(andItem, value); + + return andItem; + } + + + @NonNull + private CompositeItem buildNotAnd(String key, Inspector value) { + NotItem notItem = new NotItem(); + addItemsFromInspector(notItem, value); + + return notItem; + } + + + @NonNull + private CompositeItem buildOr(String key, Inspector value) { + OrItem orItem = new OrItem(); + addItemsFromInspector(orItem, value); + return orItem; + } + + + @NonNull + private CompositeItem buildWeakAnd(String key, Inspector value) { + WeakAndItem weakAnd = new WeakAndItem(); + addItemsFromInspector(weakAnd, value); + Inspector annotations = getAnnotations(value); + + if (annotations != null){ + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (TARGET_NUM_HITS.equals(annotation_name)){ + weakAnd.setN((int)(annotation_value.asDouble())); + } + if (SCORE_THRESHOLD.equals(annotation_name)){ + weakAnd.setScoreThreshold((int)(annotation_value.asDouble())); + } + }); + } + + return weakAnd; + } + + + @NonNull + private <T extends TaggableItem> T leafStyleSettings(Inspector annotations, @NonNull T out) { + { + if (annotations != null) { + Inspector itemConnectivity= getAnnotationAsInspectorOrNull(CONNECTIVITY, getAnnotationMapFromAnnotationInspector(annotations)); + if (itemConnectivity != null) { + Integer[] id = {null}; + Double[] weight = {null}; + itemConnectivity.traverse((ObjectTraverser) (key, value) -> { + switch (key){ + case CONNECTION_ID: + id[0] = (int) value.asLong(); + break; + case CONNECTION_WEIGHT: + weight[0] = value.asDouble(); + break; + } + }); + connectedItems.add(new ConnectedItem(out, id[0], weight[0])); + } + + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + + if (SIGNIFICANCE.equals(annotation_name)) { + if (annotation_value != null) { + out.setSignificance(annotation_value.asDouble()); + } + } + if (UNIQUE_ID.equals(annotation_name)) { + if (annotation_value != null) { + out.setUniqueID((int)annotation_value.asLong()); + identifiedItems.put((int)annotation_value.asLong(), out); + } + } + }); + } + } + { + Item leaf = (Item) out; + if (annotations != null) { + Inspector itemAnnotations = getAnnotationAsInspectorOrNull(ANNOTATIONS, getAnnotationMapFromAnnotationInspector(annotations)); + if (itemAnnotations != null) { + itemAnnotations.traverse((ObjectTraverser) (key, value) -> { + leaf.addAnnotation(key, value.asString()); + }); + } + + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (FILTER.equals(annotation_name)) { + if (annotation_value != null) { + leaf.setFilter(annotation_value.asBool()); + } + } + if (RANKED.equals(annotation_name)) { + if (annotation_value != null) { + leaf.setRanked(annotation_value.asBool()); + } + } + if (LABEL.equals(annotation_name)) { + if (annotation_value != null) { + leaf.setLabel(annotation_value.asString()); + } + } + if (WEIGHT.equals(annotation_name)) { + if (annotation_value != null) { + leaf.setWeight((int)annotation_value.asDouble()); + } + } + }); + } + if (out instanceof IntItem && annotations != null) { + IntItem number = (IntItem) out; + Integer hitLimit = getCappedRangeSearchParameter(annotations); + if (hitLimit != null) { + number.setHitLimit(hitLimit); + } + + } + } + + return out; + } + + + private Integer getCappedRangeSearchParameter(Inspector annotations) { + final Integer[] hitLimit = {null}; + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (HIT_LIMIT.equals(annotation_name)) { + if (annotation_value != null) { + hitLimit[0] = (int)(annotation_value.asDouble()); + } + } + }); + final Boolean[] ascending = {null}; + final Boolean[] descending = {null}; + + if (hitLimit[0] != null) { + annotations.traverse((ObjectTraverser) (annotation_name, annotation_value) -> { + if (ASCENDING_HITS_ORDER.equals(annotation_name)) { + ascending[0] = annotation_value.asBool(); + } + if (DESCENDING_HITS_ORDER.equals(annotation_name)) { + descending[0] = annotation_value.asBool(); + } + + }); + Preconditions.checkArgument(ascending[0] == null || descending[0] == null, + "Settings for both ascending and descending ordering set, only one of these expected."); + + if (Boolean.TRUE.equals(descending[0]) || Boolean.FALSE.equals(ascending[0])) { + hitLimit[0] = hitLimit[0] * -1; + } + } + return hitLimit[0]; + } + + + @NonNull + private Item buildRange(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + Inspector annotations = getAnnotations(value); + + final boolean[] equals = {false}; + + String field; + Inspector boundInspector; + if (children.get(0).type() == STRING){ + field = children.get(0).asString(); + boundInspector = children.get(1); + } else { + field = children.get(1).asString(); + boundInspector = children.get(0); + } + + final Number[] bounds = {null, null}; + final String[] operators = {null, null}; + boundInspector.traverse((ObjectTraverser) (operator, bound) -> { + if (bound.type() == STRING) { + throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."); + } + if (operator.equals("=")) { + bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + operators[0] = operator; + equals[0] = true; + } + if (operator.equals(">=") || operator.equals(">")){ + bounds[0] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + operators[0] = operator; + } else if (operator.equals("<=") || operator.equals("<")){ + bounds[1] = (bound.type() == DOUBLE) ? Number.class.cast(bound.asDouble()) : Number.class.cast(bound.asLong()); + operators[1] = operator; + } + + }); + IntItem range = null; + if (equals[0]){ + range = new IntItem(bounds[0].toString(), field); + } else if (operators[0]==null || operators[1]==null){ + Integer index = (operators[0] == null) ? 1 : 0; + switch (operators[index]){ + case ">=": + range = buildGreaterThanOrEquals(field, bounds[index].toString()); + break; + case ">": + range = buildGreaterThan(field, bounds[index].toString()); + break; + case "<": + range = buildLessThan(field, bounds[index].toString()); + break; + case "<=": + range = buildLessThanOrEquals(field, bounds[index].toString()); + break; + } + } + else { + range = instantiateRangeItem(bounds[0], bounds[1], field, operators[0].equals(">"), operators[1].equals("<")); + } + + return leafStyleSettings(annotations, range); + } + + @NonNull + private IntItem buildGreaterThanOrEquals(String field, String bound) { + return new IntItem("[" + bound + ";]", field); + + } + + + @NonNull + private IntItem buildLessThanOrEquals(String field, String bound) { + return new IntItem("[;" + bound + "]", field); + } + + + @NonNull + private IntItem buildGreaterThan(String field, String bound) { + return new IntItem(">" + bound, field); + + } + + + @NonNull + private IntItem buildLessThan(String field, String bound) { + return new IntItem("<" + bound, field); + } + + + @NonNull + private IntItem instantiateRangeItem(Number lowerBound, Number upperBound, String field, boolean bounds_left_open, boolean bounds_right_open) { + Preconditions.checkArgument(lowerBound != null && upperBound != null && field != null, + "Expected 3 NonNull-arguments"); + + if (!bounds_left_open && !bounds_right_open) { + return new RangeItem(lowerBound, upperBound, field); + } else { + Limit from; + Limit to; + if (bounds_left_open && bounds_right_open) { + from = new Limit(lowerBound, false); + to = new Limit(upperBound, false); + } else if (bounds_left_open) { + from = new Limit(lowerBound, false); + to = new Limit(upperBound, true); + } else { + from = new Limit(lowerBound, true); + to = new Limit(upperBound, false); + } + return new IntItem(from, to, field); + } + } + + + @NonNull + private Item buildEquals(String key, Inspector value) { + return buildRange(key, value); + } + + + @NonNull + private Item buildWand(String key, Inspector value) { + HashMap<String, Inspector> annotations = getAnnotationMap(value); + HashMap<Integer, Inspector> children = getChildrenMap(value); + + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + Integer target_num_hits= getIntegerAnnotation(TARGET_NUM_HITS, annotations, DEFAULT_TARGET_NUM_HITS); + + WandItem out = new WandItem(children.get(0).asString(), target_num_hits); + + Double scoreThreshold = getDoubleAnnotation(SCORE_THRESHOLD, annotations, null); + + if (scoreThreshold != null) { + out.setScoreThreshold(scoreThreshold); + } + + Double thresholdBoostFactor = getDoubleAnnotation(THRESHOLD_BOOST_FACTOR, annotations, null); + if (thresholdBoostFactor != null) { + out.setThresholdBoostFactor(thresholdBoostFactor); + } + return fillWeightedSet(value, children, out); + } + + + @NonNull + private WeightedSetItem fillWeightedSet(Inspector value, HashMap<Integer, Inspector> children, @NonNull WeightedSetItem out) { + addItems(children, out); + + return leafStyleSettings(getAnnotations(value), out); + } + + + private static void addItems(HashMap<Integer, Inspector> children, WeightedSetItem out) { + switch (children.get(1).type()) { + case OBJECT: + addStringItems(children, out); + break; + case ARRAY: + addLongItems(children, out); + break; + default: + throw newUnexpectedArgumentException(children.get(1).type(), ARRAY, OBJECT); + } + } + + + private static void addStringItems(HashMap<Integer, Inspector> children, WeightedSetItem out) { + //{"a":1, "b":2} + children.get(1).traverse((ObjectTraverser) (key, value) -> { + if (value.type() == STRING){ + throw new IllegalArgumentException("Expected operator LITERAL, got READ_FIELD."); + } + out.addToken(key, (int)value.asLong()); + }); + } + + + private static void addLongItems(HashMap<Integer, Inspector> children, WeightedSetItem out) { + //[[11,1], [37,2]] + children.get(1).traverse((ArrayTraverser) (index, pair) -> { + List<Integer> pairValues = new ArrayList<>(); + pair.traverse((ArrayTraverser) (pairIndex, pairValue) -> { + pairValues.add((int)pairValue.asLong()); + }); + Preconditions.checkArgument(pairValues.size() == 2, + "Expected item and weight, got %s.", pairValues); + out.addToken(pairValues.get(0).longValue(), pairValues.get(1)); + }); + } + + + @NonNull + private Item buildRegExpSearch(String key, Inspector value) { + assertHasOperator(key, MATCHES); + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + String wordData = children.get(1).asString(); + RegExpItem regExp = new RegExpItem(field, true, wordData); + return leafStyleSettings(getAnnotations(value), regExp); + } + + + @NonNull + private Item buildWeightedSet(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + return fillWeightedSet(value, children, new WeightedSetItem(field)); + } + + + @NonNull + private Item buildDotProduct(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + Preconditions.checkArgument(children.size() == 2, "Expected 2 arguments, got %s.", children.size()); + return fillWeightedSet(value, children, new DotProductItem(field)); + } + + + @NonNull + private Item buildPredicate(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + Inspector args = children.get(1); + + Preconditions.checkArgument(children.size() == 3, "Expected 3 arguments, got %s.", children.size()); + + PredicateQueryItem item = new PredicateQueryItem(); + item.setIndexName(field); + + List<Inspector> argumentList = valueListFromInspector(getChildren(value)); + + // Adding attributes + argumentList.get(1).traverse((ObjectTraverser) (attrKey, attrValue) -> { + if (attrValue.type() == ARRAY){ + List<Inspector> attributes = valueListFromInspector(attrValue); + attributes.forEach( (attribute) -> item.addFeature(attrKey, attribute.asString())); + } else { + item.addFeature(attrKey, attrValue.asString()); + } + }); + + // Adding range attributes + argumentList.get(2).traverse((ObjectTraverser) (attrKey, attrValue) -> item.addRangeFeature(attrKey, (int)attrValue.asDouble())); + + return leafStyleSettings(getAnnotations(value), item); + } + + + @NonNull + private CompositeItem buildRank(String key, Inspector value) { + RankItem rankItem = new RankItem(); + addItemsFromInspector(rankItem, value); + return rankItem; + } + + + @NonNull + private Item buildTermSearch(String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + String field = children.get(0).asString(); + + return instantiateLeafItem(field, key, value); + } + + + private String getInspectorKey(Inspector inspector){ + String[] actualKey = {""}; + if (inspector.type() == OBJECT){ + inspector.traverse((ObjectTraverser) (key, value) -> { + actualKey[0] = key; + + }); + } + return actualKey[0]; + } + + + @NonNull + private Item instantiateLeafItem(String field, String key, Inspector value) { + List<Inspector> possibleLeafFunction = valueListFromInspector(value); + String possibleLeafFunctionName = (possibleLeafFunction.size() > 1) ? getInspectorKey(possibleLeafFunction.get(1)) : ""; + if (FUNCTION_CALLS.contains(key)) { + return instantiateCompositeLeaf(field, key, value); + } else if(!possibleLeafFunctionName.equals("")){ + return instantiateCompositeLeaf(field, possibleLeafFunctionName, valueListFromInspector(value).get(1).field(possibleLeafFunctionName)); + } else { + return instantiateWordItem(field, key, value); + } + } + + + @NonNull + private Item instantiateCompositeLeaf(String field, String key, Inspector value) { + switch (key) { + case SAME_ELEMENT: + return instantiateSameElementItem(field, key, value); + case PHRASE: + return instantiatePhraseItem(field, key, value); + case NEAR: + return instantiateNearItem(field, key, value); + case ONEAR: + return instantiateONearItem(field, key, value); + case EQUIV: + return instantiateEquivItem(field, key, value); + case ALTERNATIVES: + return instantiateWordAlternativesItem(field, key, value); + default: + throw newUnexpectedArgumentException(key, EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT); + } + } + + + @NonNull + private Item instantiateWordItem(String field, String key, Inspector value) { + String wordData = getChildrenMap(value).get(1).asString(); + return instantiateWordItem(field, wordData, key, value, false, decideParsingLanguage(value, wordData)); + } + + + @NonNull + private Item instantiateWordItem(String field, String rawWord, String key, Inspector value, boolean exactMatch, Language language) { + String wordData = rawWord; + HashMap<String, Inspector> annotations = getAnnotationMap(value); + + if (getBoolAnnotation(NFKC, annotations, Boolean.FALSE)) { + // NOTE: If this is set to FALSE (default), we will still NFKC normalize text data + // during tokenization/segmentation, as that is always turned on also on the indexing side. + wordData = normalizer.normalize(wordData); + } + boolean fromQuery = getBoolAnnotation(IMPLICIT_TRANSFORMS, annotations, Boolean.TRUE); + boolean prefixMatch = getBoolAnnotation(PREFIX, annotations, Boolean.FALSE); + boolean suffixMatch = getBoolAnnotation(SUFFIX, annotations, Boolean.FALSE); + boolean substrMatch = getBoolAnnotation(SUBSTRING,annotations, Boolean.FALSE); + + Preconditions.checkArgument((prefixMatch ? 1 : 0) + + (substrMatch ? 1 : 0) + (suffixMatch ? 1 : 0) < 2, + "Only one of prefix, substring and suffix can be set."); + @NonNull + final TaggableItem wordItem; + + if (exactMatch) { + wordItem = new ExactStringItem(wordData, fromQuery); + } else if (prefixMatch) { + wordItem = new PrefixItem(wordData, fromQuery); + } else if (suffixMatch) { + wordItem = new SuffixItem(wordData, fromQuery); + } else if (substrMatch) { + wordItem = new SubstringItem(wordData, fromQuery); + } else { + wordItem = new WordItem(wordData, fromQuery); + } + + if (wordItem instanceof WordItem) { + prepareWord(field, value, (WordItem) wordItem); + } + if (language != Language.ENGLISH) + ((Item)wordItem).setLanguage(language); + + return (Item) leafStyleSettings(getAnnotations(value), wordItem); + } + + + private Language decideParsingLanguage(Inspector value, String wordData) { + String languageTag = getAnnotation(USER_INPUT_LANGUAGE, getAnnotationMap(value), String.class, null); + + Language language = Language.fromLanguageTag(languageTag); + if (language != Language.UNKNOWN) return language; + + Optional<Language> explicitLanguage = query.getExplicitLanguage(); + if (explicitLanguage.isPresent()) return explicitLanguage.get(); + + return Language.ENGLISH; + } + + + private void prepareWord(String field, Inspector value, WordItem wordItem) { + wordItem.setIndexName(field); + wordStyleSettings(value, wordItem); + } + + + private void wordStyleSettings(Inspector value, WordItem out) { + HashMap<String, Inspector> annotations = getAnnotationMap(value); + + Substring origin = getOrigin(getAnnotations(value)); + if (origin != null) { + out.setOrigin(origin); + } + if (annotations != null){ + Boolean usePositionData = Boolean.getBoolean(getAnnotation(USE_POSITION_DATA, annotations, String.class, null)); + if (usePositionData != null) { + out.setPositionData(usePositionData); + } + Boolean stem = getBoolAnnotation(STEM, annotations, null); + if (stem != null) { + out.setStemmed(!stem); + } + + Boolean normalizeCase = getBoolAnnotation(NORMALIZE_CASE, annotations, null); + if (normalizeCase != null) { + out.setLowercased(!normalizeCase); + } + Boolean accentDrop = getBoolAnnotation(ACCENT_DROP, annotations, null); + if (accentDrop != null) { + out.setNormalizable(accentDrop); + } + Boolean andSegmenting = getBoolAnnotation(AND_SEGMENTING, annotations, null); + if (andSegmenting != null) { + if (andSegmenting) { + out.setSegmentingRule(SegmentingRule.BOOLEAN_AND); + } else { + out.setSegmentingRule(SegmentingRule.PHRASE); + } + } + } + } + + + private Substring getOrigin(Inspector annotations) { + if (annotations != null) { + Inspector origin = getAnnotationAsInspectorOrNull(ORIGIN, getAnnotationMapFromAnnotationInspector(annotations)); + if (origin == null) { + return null; + } + final String[] original = {null}; + final Integer[] offset = {null}; + final Integer[] length = {null}; + + origin.traverse((ObjectTraverser) (key, value) -> { + switch (key) { + case (ORIGIN_ORIGINAL): + original[0] = value.asString(); + break; + case (ORIGIN_OFFSET): + offset[0] = (int) value.asDouble(); + break; + case (ORIGIN_LENGTH): + length[0] = (int) value.asDouble(); + break; + } + + + }); + return new Substring(offset[0], length[0] + offset[0], original[0]); + } + return null; + } + + + @NonNull + private Item instantiateSameElementItem(String field, String key, Inspector value) { + assertHasOperator(key, SAME_ELEMENT); + + SameElementItem sameElement = new SameElementItem(field); + // All terms below sameElement are relative to this. + getChildren(value).traverse((ArrayTraverser) (index, term) -> { + sameElement.addItem(walkJson(term)); + }); + + return sameElement; + } + + + @NonNull + private Item instantiatePhraseItem(String field, String key, Inspector value) { + assertHasOperator(key, PHRASE); + HashMap<String, Inspector> annotations = getAnnotationMap(value); + + PhraseItem phrase = new PhraseItem(); + phrase.setIndexName(field); + HashMap<Integer, Inspector> children = getChildrenMap(value); + + for (Inspector word : children.values()) + if (word.type() == STRING) phrase.addItem(new WordItem(word.asString())); + else if (word.type() == OBJECT && word.field(PHRASE).valid()) { + phrase.addItem(instantiatePhraseItem(field, key, getChildren(word))); + } + return leafStyleSettings(getAnnotations(value), phrase); + } + + + @NonNull + private Item instantiateNearItem(String field, String key, Inspector value) { + assertHasOperator(key, NEAR); + + NearItem near = new NearItem(); + near.setIndexName(field); + + HashMap<Integer, Inspector> children = getChildrenMap(value); + + for (Inspector word : children.values()){ + near.addItem(new WordItem(word.asString(), field)); + } + + Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null); + + if (distance != null) { + near.setDistance((int)distance); + } + return near; + } + + + @NonNull + private Item instantiateONearItem(String field, String key, Inspector value) { + assertHasOperator(key, ONEAR); + + NearItem onear = new ONearItem(); + onear.setIndexName(field); + HashMap<Integer, Inspector> children = getChildrenMap(value); + + for (Inspector word : children.values()){ + onear.addItem(new WordItem(word.asString(), field)); + } + + Integer distance = getIntegerAnnotation(DISTANCE, getAnnotationMap(value), null); + if (distance != null) { + onear.setDistance(distance); + } + return onear; + } + + + @NonNull + private Item instantiateEquivItem(String field, String key, Inspector value) { + + HashMap<Integer, Inspector> children = getChildrenMap(value); + Preconditions.checkArgument(children.size() >= 2, "Expected 2 or more arguments, got %s.", children.size()); + + EquivItem equiv = new EquivItem(); + equiv.setIndexName(field); + + for (Inspector word : children.values()){ + if (word.type() == STRING || word.type() == LONG || word.type() == DOUBLE){ + equiv.addItem(new WordItem(word.asString(), field)); + } + if (word.type() == OBJECT){ + word.traverse((ObjectTraverser) (key2, value2) -> { + assertHasOperator(key2, PHRASE); + equiv.addItem(instantiatePhraseItem(field, key2, value2)); + }); + } + } + + return leafStyleSettings(getAnnotations(value), equiv); + } + + + private Item instantiateWordAlternativesItem(String field, String key, Inspector value) { + HashMap<Integer, Inspector> children = getChildrenMap(value); + Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size()); + Preconditions.checkArgument(children.get(0).type() == OBJECT, "Expected OBJECT, got %s.", children.get(0).type()); + + List<WordAlternativesItem.Alternative> terms = new ArrayList<>(); + + children.get(0).traverse((ObjectTraverser) (keys, values) -> { + terms.add(new WordAlternativesItem.Alternative(keys, values.asDouble())); + }); + return leafStyleSettings(getAnnotations(value), new WordAlternativesItem(field, Boolean.TRUE, null, terms)); + } + + + // Not in use yet + @NonNull + private String getIndex(String field) { + Preconditions.checkArgument(indexFactsSession.isIndex(field), "Field '%s' does not exist.", field); + //return indexFactsSession.getCanonicName(field); + return field; + } + + + private static void assertHasOperator(String key, String expectedKey) { + Preconditions.checkArgument(key.equals(expectedKey), "Expected operator %s, got %s.", expectedKey, key); + } + + + private static IllegalArgumentException newUnexpectedArgumentException(Object actual, Object... expected) { + StringBuilder out = new StringBuilder("Expected "); + for (int i = 0, len = expected.length; i < len; ++i) { + out.append(expected[i]); + if (i < len - 2) { + out.append(", "); + } else if (i < len - 1) { + out.append(" or "); + } + } + out.append(", got ").append(actual).append("."); + return new IllegalArgumentException(out.toString()); + } + + + private List<Inspector> valueListFromInspector(Inspector inspector){ + List<Inspector> inspectorList = new ArrayList<>(); + inspector.traverse((ArrayTraverser) (key, value) -> inspectorList.add(value)); + return inspectorList; + } + + + private void connectItems() { + for (ConnectedItem entry : connectedItems) { + TaggableItem to = identifiedItems.get(entry.toId); + Preconditions.checkNotNull(to, + "Item '%s' was specified to connect to item with ID %s, which does not " + + "exist in the query.", entry.fromItem, + entry.toId); + entry.fromItem.setConnectivity((Item) to, entry.weight); + } + } + + + private static final class ConnectedItem { + + final double weight; + final int toId; + final TaggableItem fromItem; + + ConnectedItem(TaggableItem fromItem, int toId, double weight) { + this.weight = weight; + this.toId = toId; + this.fromItem = fromItem; + } + } + + +} diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java index e5941a90b83..64fb201fe21 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/Parsable.java @@ -3,6 +3,7 @@ package com.yahoo.search.query.parser; import com.yahoo.language.Language; import com.yahoo.search.query.Model; +import com.yahoo.search.query.Select; import java.util.Collection; import java.util.HashSet; @@ -36,6 +37,7 @@ public final class Parsable { private String defaultIndexName; private Language language; // TODO: Initialize to UNKNOWN private Optional<Language> explicitLanguage = Optional.empty(); + private Select select; /** If this is set it will be used to determine the language, if not set explicitly */ private Optional<Model> model = Optional.empty(); @@ -133,6 +135,15 @@ public final class Parsable { return this; } + public Parsable setSelect(Select select){ + this.select = select; + return this; + } + + public Select getSelect(){ + return this.select; + } + public static Parsable fromQueryModel(Model model) { return new Parsable() .setModel(model) @@ -141,7 +152,11 @@ public final class Parsable { .setExplicitLanguage(Optional.ofNullable(model.getLanguage())) .setDefaultIndexName(model.getDefaultIndex()) .addSources(model.getSources()) - .addRestricts(model.getRestrict()); + .addRestricts(model.getRestrict()) + .setSelect(model.getParent().getSelect()); } + + + } diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java index 8d008abaac2..69d46527255 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java @@ -3,6 +3,7 @@ package com.yahoo.search.query.parser; import com.yahoo.prelude.query.parser.*; import com.yahoo.search.Query; +import com.yahoo.search.query.SelectParser; import com.yahoo.search.yql.YqlParser; /** @@ -40,6 +41,8 @@ public final class ParserFactory { return new ProgrammaticParser(); case YQL: return new YqlParser(environment); + case SELECT: + return new SelectParser(environment); default: throw new UnsupportedOperationException(type.toString()); } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 0aea5e96161..71002166b11 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -113,6 +113,10 @@ public class QueryProperties extends Properties { if (key.get(1).equals(Ranking.PROPERTIES)) return ranking.getProperties().get(key.rest().rest().toString()); } } + else if (key.size()==2 && key.first().equals(Select.SELECT)) { + if (key.last().equals(Select.WHERE)) return query.getSelect().getWhereString(); + if (key.last().equals(Select.GROUPING)) return query.getSelect().getGroupingString(); + } else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) { if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); @@ -247,6 +251,13 @@ public class QueryProperties extends Properties { else if ( ! key.last().equals(Presentation.REPORT_COVERAGE)) // TODO: Change this line to "else" on Vespa 7 throwIllegalParameter(key.last(), Presentation.PRESENTATION); } + else if (key.size()==2 && key.first().equals(Select.SELECT)) { + if (key.last().equals(Select.WHERE)){ + query.getSelect().setWhere(asString(value, "")); + } else if (key.last().equals(Select.GROUPING)) { + query.getSelect().setGrouping(asString(value, "")); + } + } else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated setRankingFeature(query, key.rest().toString(), toSpecifiedType(key.rest().toString(), value, profileRegistry.getTypeRegistry().getComponent("features"))); } else if (key.first().equals("rankproperty")) { diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java b/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java index 1b50ae20183..ec79979387b 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/PhaseNames.java @@ -7,7 +7,7 @@ package com.yahoo.search.searchchain; * a searcher should depend on some explicit functionality, not these * checkpoints. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public final class PhaseNames { private PhaseNames() { diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 6bad032600c..e0e9042e1a3 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -789,7 +789,7 @@ public class YqlParser implements Parser { OperatorNode<ExpressionOperator> groupingAst = ast.<List<OperatorNode<ExpressionOperator>>> getArgument(2).get(0); GroupingOperation groupingOperation = GroupingOperation.fromString(groupingAst.<String> getArgument(0)); VespaGroupingStep groupingStep = new VespaGroupingStep(groupingOperation); - List<String> continuations = getAnnotation(groupingAst, "continuations", List.class, + List<String> continuations = getAnnotation(groupingAst, "continuations", List.class, Collections.emptyList(), "grouping continuations"); for (String continuation : continuations) { groupingStep.continuations().add(Continuation.fromString(continuation)); diff --git a/container-search/src/main/resources/configdefinitions/provider.def b/container-search/src/main/resources/configdefinitions/provider.def index 79b09913b49..f9ab305b114 100644 --- a/container-search/src/main/resources/configdefinitions/provider.def +++ b/container-search/src/main/resources/configdefinitions/provider.def @@ -35,7 +35,7 @@ yca.ttl int default=0 yca.retry int default=0 # The form of the serialized query. -queryType enum { LEGACY, PROGRAMMATIC, YQL } default=LEGACY +queryType enum { LEGACY, PROGRAMMATIC, YQL, SELECT } default=LEGACY # How to do pinging against a backend. pingOption enum { DISABLE, NORMAL, YCA } default=NORMAL |