// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search; import ai.vespa.cloud.ZoneInfo; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.fs4.MapEncoder; import com.yahoo.language.process.Embedder; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.textualrepresentation.TextualQueryRepresentation; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.dispatch.rpc.ProtobufSerialization; import com.yahoo.search.federation.FederationSearcher; import com.yahoo.search.query.Model; import com.yahoo.search.query.ParameterParser; import com.yahoo.search.query.Presentation; import com.yahoo.search.query.Properties; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.Select; import com.yahoo.search.query.SessionId; import com.yahoo.search.query.Sorting; import com.yahoo.search.query.Sorting.AttributeSorter; import com.yahoo.search.query.Sorting.FieldOrder; import com.yahoo.search.query.Sorting.Order; import com.yahoo.search.query.UniqueRequestId; import com.yahoo.search.query.context.QueryContext; import com.yahoo.search.query.profile.ModelObjectMap; import com.yahoo.search.query.profile.QueryProfileProperties; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; import com.yahoo.search.query.profile.types.QueryProfileFieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; import com.yahoo.search.query.properties.DefaultProperties; import com.yahoo.search.query.properties.PropertyMap; import com.yahoo.search.query.properties.QueryProperties; import com.yahoo.search.query.properties.QueryPropertyAliases; import com.yahoo.search.query.properties.RequestContextProperties; import com.yahoo.search.yql.NullItemException; import com.yahoo.search.yql.VespaSerializer; import com.yahoo.search.yql.YqlParser; import com.yahoo.yolean.Exceptions; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; /** * A search query containing all the information required to produce a Result. *
* The Query contains: *
* The properties has three sources *
* The identity of a query is determined by its content.
*
* @author Arne Bergene Fossaa
* @author bratseth
*/
public class Query extends com.yahoo.processing.Request implements Cloneable {
// Note to developers: If you think you should add something here you are probably wrong
// To add state to the query: Do properties.set("myNewState",new MyNewState()) instead.
/** The type of the query */
public enum Type {
ALL(0,"all"),
ANY(1,"any"),
PHRASE(2,"phrase"),
ADVANCED(3,"adv"),
WEB(4,"web"),
PROGRAMMATIC(5, "prog"),
YQL(6, "yql"),
SELECT(7, "select"),
WEAKAND(8, "weakAnd"),
TOKENIZE(9, "tokenize");
private final int intValue;
private final String stringValue;
Type(int intValue, String stringValue) {
this.intValue = intValue;
this.stringValue = stringValue;
}
/** Converts a type argument value into a query type */
public static Type getType(String typeString) {
for (Type type : Type.values())
if (type.stringValue.equals(typeString))
return type;
throw new IllegalArgumentException("No query type '" + typeString + "'");
}
public int asInt() { return intValue; }
public String toString() { return stringValue; }
}
//-------------- Query properties treated as fields in Query ---------------
/** The offset from the most relevant hits found from this query */
private int offset = 0;
/** The number of hits to return */
private int hits = 10;
/** The query context level, 0 means no tracing */
private int traceLevel = 0;
/** The query explain level, 0 means no explaining */
private int explainLevel = 0;
// The timeout to be used when dumping rank features
private static final long dumpTimeout = (6 * 60 * 1000); // 6 minutes
private static final long defaultTimeout = 500;
/** The timeout of the query, in milliseconds */
private long timeout = defaultTimeout;
/** Whether this query is forbidden to access cached information */
private boolean noCache = false;
/** Whether or not grouping should use a session cache */
private boolean groupingSessionCache = true;
//-------------- Generic property containers --------------------------------
/** The synchronous view of the JDisc request causing this query */
private final HttpRequest httpRequest;
/** The context, or null if there is no context */
private QueryContext context = null;
/** Used for downstream session caches */
private UniqueRequestId requestId = null;
//--------------- Owned sub-objects containing query properties ----------------
/** The ranking requested in this query */
private Ranking ranking = new Ranking(this);
/** The query query and/or query program declaration */
private Model model = new Model(this);
/** How results of this query should be presented */
private Presentation presentation = new Presentation(this);
/** The selection of where-clause and grouping */
private Select select = new Select(this);
//---------------- Tracing ----------------------------------------------------
private static final Logger log = Logger.getLogger(Query.class.getName());
/** The time this query was created */
private long startTime;
//---------------- Static property handling ------------------------------------
public static final CompoundName OFFSET = new CompoundName("offset");
public static final CompoundName HITS = new CompoundName("hits");
public static final CompoundName QUERY_PROFILE = new CompoundName("queryProfile");
public static final CompoundName SEARCH_CHAIN = new CompoundName("searchChain");
public static final CompoundName TRACE_LEVEL = new CompoundName("traceLevel");
public static final CompoundName EXPLAIN_LEVEL = new CompoundName("explainLevel");
public static final CompoundName NO_CACHE = new CompoundName("noCache");
public static final CompoundName GROUPING_SESSION_CACHE = new CompoundName("groupingSessionCache");
public static final CompoundName TIMEOUT = new CompoundName("timeout");
private static final QueryProfileType argumentType;
static {
argumentType = new QueryProfileType("native");
argumentType.setBuiltin(true);
argumentType.addField(new FieldDescription(OFFSET.toString(), "integer", "offset start"));
argumentType.addField(new FieldDescription(HITS.toString(), "integer", "hits count"));
argumentType.addField(new FieldDescription(QUERY_PROFILE.toString(), "string"));
argumentType.addField(new FieldDescription(SEARCH_CHAIN.toString(), "string"));
argumentType.addField(new FieldDescription(TRACE_LEVEL.toString(), "integer", "tracelevel"));
argumentType.addField(new FieldDescription(EXPLAIN_LEVEL.toString(), "integer", "explainlevel"));
argumentType.addField(new FieldDescription(NO_CACHE.toString(), "boolean", "nocache"));
argumentType.addField(new FieldDescription(GROUPING_SESSION_CACHE.toString(), "boolean", "groupingSessionCache"));
argumentType.addField(new FieldDescription(TIMEOUT.toString(), "string", "timeout"));
argumentType.addField(new FieldDescription(FederationSearcher.SOURCENAME.toString(),"string"));
argumentType.addField(new FieldDescription(FederationSearcher.PROVIDERNAME.toString(),"string"));
argumentType.addField(new FieldDescription(Presentation.PRESENTATION, new QueryProfileFieldType(Presentation.getArgumentType())));
argumentType.addField(new FieldDescription(Ranking.RANKING, new QueryProfileFieldType(Ranking.getArgumentType())));
argumentType.addField(new FieldDescription(Model.MODEL, new QueryProfileFieldType(Model.getArgumentType())));
argumentType.addField(new FieldDescription(Select.SELECT, new QueryProfileFieldType(Select.getArgumentType())));
argumentType.addField(new FieldDescription(Dispatcher.DISPATCH, new QueryProfileFieldType(Dispatcher.getArgumentType())));
argumentType.freeze();
}
public static QueryProfileType getArgumentType() { return argumentType; }
/** The aliases of query properties */
private static final Map
* Note: If Ranking.RANKFEATURES is turned on, this is hardcoded to 6 minutes.
*
* @return timeout in milliseconds.
*/
public long getTimeout() {
return properties().getBoolean(Ranking.RANKFEATURES, false) ? dumpTimeout : timeout;
}
/**
* Sets the number of milliseconds to wait for a response from a search backend
* before time out. Default is 500 ms.
*/
public void setTimeout(long timeout) {
if (timeout > 1000000000 || timeout < 0)
throw new IllegalArgumentException("'timeout' must be positive and smaller than 1000000000 ms but was " + timeout);
this.timeout = timeout;
}
/**
* Sets timeout from a string which will be parsed as a
*/
public void setTimeout(String timeoutString) {
setTimeout(ParameterParser.asMilliSeconds(timeoutString, timeout));
}
/**
* Resets the start time of the query. This will ensure that the query will run
* for the same amount of time as a newly created query.
*/
public void resetTimeout() { this.startTime = System.currentTimeMillis(); }
/**
* Sets the context level of this query, 0 means no tracing
* Higher numbers means increasingly more tracing
*/
public void setTraceLevel(int traceLevel) { this.traceLevel = traceLevel; }
/**
* Sets the explain level of this query, 0 means no tracing
* Higher numbers means increasingly more explaining
*/
public void setExplainLevel(int explainLevel) { this.explainLevel = explainLevel; }
/**
* Returns the context level of this query, 0 means no tracing
* Higher numbers means increasingly more tracing
*/
public int getTraceLevel() { return traceLevel; }
/**
* Returns the explain level of this query, 0 means no tracing
* Higher numbers means increasingly more explaining
*/
public int getExplainLevel() { return explainLevel; }
/**
* Returns the context level of this query, 0 means no tracing
* Higher numbers means increasingly more tracing
*/
public final boolean isTraceable(int level) { return traceLevel >= level; }
/** Returns whether this query should never be served from a cache. Default is false */
public boolean getNoCache() { return noCache; }
/** Sets whether this query should never be server from a cache. Default is false */
public void setNoCache(boolean noCache) { this.noCache = noCache; }
/** Returns whether this query should use the grouping session cache. Default is false */
public boolean getGroupingSessionCache() { return groupingSessionCache; }
/** Sets whether this query should use the grouping session cache. Default is false */
public void setGroupingSessionCache(boolean groupingSessionCache) { this.groupingSessionCache = groupingSessionCache; }
/**
* Returns the offset from the most relevant hits requested by the submitter
* of this query.
* Default is 0 - to return the most relevant hits
*/
public int getOffset() { return offset; }
/**
* Returns the number of hits requested by the submitter of this query.
* The default is 10.
*/
public int getHits() { return hits; }
/**
* Sets the number of hits requested. If hits is less than 0, an
* IllegalArgumentException is thrown. Default number of hits is 10.
*/
public void setHits(int hits) {
if (hits < 0)
throw new IllegalArgumentException("Must be a positive number");
this.hits = hits;
}
/**
* Set the hit offset. Can not be less than 0. Default is 0.
*/
public void setOffset(int offset) {
if (offset < 0)
throw new IllegalArgumentException("Must be a positive number");
this.offset = offset;
}
/** Convenience method to set both the offset and the number of hits to return */
public void setWindow(int offset,int hits) {
setOffset(offset);
setHits(hits);
}
/** Returns a string describing this query */
@Override
public String toString() {
String queryTree;
// getQueryTree isn't exception safe
try {
queryTree = model.getQueryTree().toString();
} catch (Exception | StackOverflowError e) {
queryTree = "[Could not parse user input: " + model.getQueryString() + "]";
}
return "query '" + queryTree + "'";
}
/** Returns a string describing this query in more detail */
public String toDetailString() {
String queryTree;
// getQueryTree isn't exception safe
try {
queryTree = model.getQueryTree().toString();
} catch (Exception | StackOverflowError e) {
queryTree = "Could not parse user input: " + model.getQueryString();
}
return "query=[" + queryTree + "]" + " offset=" + getOffset() + " hits=" + getHits() + "]";
}
/**
* Encodes this query onto the given buffer
*
* @param buffer The buffer to encode the query to
* @return the number of encoded items
*/
public int encode(ByteBuffer buffer) {
return model.getQueryTree().encode(buffer);
}
/**
* Adds a context message to this query and to the info log,
* if the context level of the query is sufficiently high.
* The context information will be carried over to the result at creation.
* The message parameter will be included with XML escaping.
*
* @param message the message to add
* @param traceLevel the context level of the message, this method will do nothing
* if the traceLevel of the query is lower than this value
*/
public void trace(String message, int traceLevel) {
trace(message, false, traceLevel);
}
public void trace(Object message, int traceLevel) {
if ( ! isTraceable(traceLevel)) return;
getContext(true).trace(message, 0);
}
/**
* Adds a trace message to this query
* if the trace level of the query is sufficiently high.
*
* @param message the message to add
* @param includeQuery true to append the query root stringValue at the end of the message
* @param traceLevel the context level of the message, this method will do nothing
* if the traceLevel of the query is lower than this value
*/
public void trace(String message, boolean includeQuery, int traceLevel) {
if ( ! isTraceable(traceLevel)) return;
if (includeQuery)
message += ": [" + queryTreeText() + "]";
log.log(Level.FINE,message);
// Pass 0 as traceLevel as the trace level check is already done above,
// and it is not propagated to trace until execution has started
// (it is done in the execution.search method)
getContext(true).trace(message, 0);
}
/**
* Adds a trace message to this query
* if the trace level of the query is sufficiently high.
*
* @param includeQuery true to append the query root stringValue at the end of the message
* @param traceLevel the context level of the message, this method will do nothing
* if the traceLevel of the query is lower than this value
* @param messages the messages whose toStrings will be concatenated into the trace message.
* Concatenation will only happen if the trace level is sufficiently high.
*/
public void trace(boolean includeQuery, int traceLevel, Object... messages) {
if ( ! isTraceable(traceLevel)) return;
StringBuilder concatenated = new StringBuilder();
for (Object message : messages)
concatenated.append(message);
trace(concatenated.toString(), includeQuery, traceLevel);
}
/**
* Set the context information for another query to be part of this query's
* context information. This is to be used if creating fresh query objects as
* part of a plug-in's execution. The query should be attached before it is
* used, in case an exception causes premature termination. This is enforced
* by an IllegalStateException. In other words, intended use is create the
* new query, and attach the context to the invoking query as soon as the new
* query is properly initialized.
*
*
* This method will always set the argument query's context level to the context
* level of this query.
*
* @param query
* The query which should be traced as a part of this query.
* @throws IllegalStateException
* If the query given as argument already has context
* information.
*/
public void attachContext(Query query) throws IllegalStateException {
query.setTraceLevel(getTraceLevel());
query.setExplainLevel(getExplainLevel());
if (context == null) {
// Nothing to attach to. This is about the same as
// getTraceLevel() == 0,
// but is a direct test of what will make the function superfluous.
return;
}
if (query.getContext(false) != null) {
// If we added the other query's context info as a subnode in this
// query's context tree, we would have to check for loops in the
// context graph. If we simply created a new node without checking,
// we might silently overwrite useful information.
throw new IllegalStateException("Query to attach already has context information stored.");
}
query.context = context;
}
private String queryTreeText() {
QueryTree root = getModel().getQueryTree();
if (getTraceLevel() < 2)
return root.toString();
if (getTraceLevel() < 6)
return yqlRepresentation();
else
return "\n" + yqlRepresentation() + "\n" + new TextualQueryRepresentation(root.getRoot()) + "\n";
}
/**
* Serialize this query as YQL+. This method will never throw exceptions,
* but instead return a human readable error message if a problem occurred while
* serializing the query. Hits and offset information will be included if
* different from default, while linguistics metadata are not added.
*
* @return a valid YQL+ query string or a human readable error message
* @see Query#yqlRepresentation(boolean)
*/
public String yqlRepresentation() {
try {
return yqlRepresentation(true);
} catch (NullItemException e) {
return "Query currently a placeholder, NullItem encountered.";
} catch (IllegalArgumentException e) {
return "Invalid query: " + Exceptions.toMessageString(e);
} catch (RuntimeException e) {
return "Unexpected error parsing or serializing query: " + Exceptions.toMessageString(e);
}
}
private void commaSeparated(StringBuilder yql, Set?query=test&offset=10&hits=13
* The query must be uri encoded.
*/
public Query(String query) {
this(query, null);
}
/**
* Creates a query from a request
*
* @param request the HTTP request from which this is created
*/
public Query(HttpRequest request) {
this(request, null);
}
/**
* Construct a query from a string formatted in the http style, e.g ?query=test&offset=10&hits=13
* The query must be uri encoded.
*/
public Query(String query, CompiledQueryProfile queryProfile) {
this(HttpRequest.createTestRequest(query, com.yahoo.jdisc.http.HttpRequest.Method.GET), queryProfile);
}
/**
* Creates a query from a request
*
* @param request the HTTP request from which this is created
* @param queryProfile the query profile to use for this query, or null if none
*/
public Query(HttpRequest request, CompiledQueryProfile queryProfile) {
this(request, request.propertyMap(), queryProfile);
}
/**
* Creates a query from a request
*
* @param request the HTTP request from which this is created
* @param requestMap the property map of the query
* @param queryProfile the query profile to use for this query, or null if none
*/
public Query(HttpRequest request, Map