diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search')
52 files changed, 104 insertions, 5316 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index bf0920a2aa5..73335037b94 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -139,7 +139,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { // The timeout to be used when dumping rank features private static final long dumpTimeout = (6 * 60 * 1000); // 6 minutes - private static final long defaultTimeout = 5000; + private static final long defaultTimeout = 500; /** The timeout of the query, in milliseconds */ private long timeout = defaultTimeout; @@ -527,17 +527,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { } /** - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public boolean requestHasProperty(String name) { - return httpRequest.hasProperty(name); - } - - /** * Returns the number of milliseconds to wait for a response from a search backend - * before timing it out. Default is 5000. + * before timing it out. Default is 500. * <p> * Note: If Ranking.RANKFEATURES is turned on, this is hardcoded to 6 minutes. * @@ -549,7 +540,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** * Sets the number of milliseconds to wait for a response from a search backend - * before time out. Default is 5000. + * before time out. Default is 500. */ public void setTimeout(long timeout) { if (timeout > 1000000000 || timeout < 0) @@ -639,24 +630,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { setHits(hits); } - /** - * This is ignored - compression is controlled at the network level. - * - * @deprecated this is ignored - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setCompress(boolean ignored) { } - - /** - * Returns false. - * - * @deprecated this always returns false - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public boolean getCompress() { return false; } - /** Returns a string describing this query */ @Override public String toString() { @@ -885,7 +858,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { yql.append(" limit ").append(Integer.toString(getHits())); } } - if (getTimeout() != 5000L) { + if (getTimeout() != defaultTimeout) { yql.append(" timeout ").append(Long.toString(getTimeout())); } yql.append(';'); @@ -1009,22 +982,6 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { */ public HttpRequest getHttpRequest() { return httpRequest; } - /** - * Returns the unique and stable session id of this query. - * - * @param create if true this is created if not already set - * @return the session id of this query, or null if not set and create is false - * @deprecated use getSessionId() or getSessionId(serverId) instead - */ - @Deprecated - public SessionId getSessionId(boolean create) { - if ( ! create) return getSessionId(); - - if (requestId == null) - requestId = UniqueRequestId.next(); - return new SessionId(requestId, getRanking().getProfile()); - } - /** Returns the session id of this query, or null if none is assigned */ public SessionId getSessionId() { if (requestId == null) return null; diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index e70916369a3..364e60e6263 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -48,11 +48,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone */ private ListMap<String,String> headers = null; - /** - * Result rendering infrastructure. - */ - private final Templating templating; - /** Creates a new Result where the top level hit group has id "toplevel" */ public Result(Query query) { this(query, new HitGroup("toplevel")); @@ -75,7 +70,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone if (query.getRanking().getSorting() != null) { setHitOrderer(new HitSortOrderer(query.getRanking().getSorting())); } - templating = new Templating(this); } /** Create a result containing an error */ @@ -92,8 +86,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone */ @SuppressWarnings("deprecation") public void mergeWith(Result result) { - if (templating.usesDefaultTemplate()) - templating.setRenderer(result.templating.getRenderer()); totalHitCount += result.getTotalHitCount(); deepHitCount += result.getDeepHitCount(); timeAccountant.merge(result.getElapsedTime()); @@ -228,7 +220,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone resultClone.hits = hits.clone(); - resultClone.getTemplating().setRenderer(null); // TODO: Remove on Vespa 7 resultClone.setElapsedTime(new ElapsedTime()); return resultClone; } @@ -299,9 +290,7 @@ public final class Result extends com.yahoo.processing.Response implements Clone * @param hit * the hit to be analyzed */ - protected void traceExtraHitProperties(StringBuilder hitBuffer, Hit hit) { - return; - } + protected void traceExtraHitProperties(StringBuilder hitBuffer, Hit hit) { } /** Returns the context of this result - this is equal to getQuery().getContext(create) */ public QueryContext getContext(boolean create) { return getQuery().getContext(create); } @@ -337,17 +326,4 @@ public final class Result extends com.yahoo.processing.Response implements Clone return headers; } - /** - * The Templating object contains helper methods and data containers for - * result rendering. - * - * @return helper object for result rendering - * @deprecated use renderers - */ - // TODO: Remove on Vespa 7 - @Deprecated// OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) - public Templating getTemplating() { - return templating; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 5de0c5eff74..ac0c8375f04 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -35,13 +35,6 @@ public class ClusterMonitor<T> { /** A map from Node to corresponding MonitoredNode */ private final Map<T, BaseNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); - /** @deprecated use the constructor with just the first argument instead */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public ClusterMonitor(NodeManager<T> manager, String ignored) { - this(manager); - } - public ClusterMonitor(NodeManager<T> manager) { nodeManager = manager; monitorThread = new MonitorThread("search.clustermonitor"); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java index fb72e8c77c9..578c447dfbe 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/RpcFillInvoker.java @@ -110,7 +110,7 @@ public class RpcFillInvoker extends FillInvoker { Query query = result.getQuery(); String rankProfile = query.getRanking().getProfile(); byte[] serializedSlime = BinaryFormat - .encode(toSlime(rankProfile, summaryClass, query.getModel().getDocumentDb(), query.getSessionId(false), hits)); + .encode(toSlime(rankProfile, summaryClass, query.getModel().getDocumentDb(), query.getSessionId(), hits)); double timeoutSeconds = ((double) query.getTimeLeft() - 3.0) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(compression, serializedSlime); resourcePool.client().getDocsums(hits, node, compressionResult.type(), serializedSlime.length, compressionResult.data(), diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java b/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java index 8ab80ec17dd..bf626312b13 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/RpcResourcePool.java @@ -3,7 +3,6 @@ package com.yahoo.search.dispatch; import com.google.common.collect.ImmutableMap; import com.yahoo.compress.Compressor; -import com.yahoo.container.search.LegacyEmulationConfig; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.fastsearch.VespaBackEndSearcher; import com.yahoo.processing.request.CompoundName; @@ -52,9 +51,8 @@ public class RpcResourcePool { if (query.properties().getBoolean(dispatchSummaries, true) && ! searcher.summaryNeedsQuery(query) && query.getRanking().getLocation() == null - && ! searcher.getCacheControl().useCache(query) - && ! legacyEmulationConfigIsSet(documentDb)) { - + && ! searcher.getCacheControl().useCache(query)) + { return Optional.of(new RpcFillInvoker(this, documentDb)); } else { return Optional.empty(); @@ -66,17 +64,6 @@ public class RpcResourcePool { return new RpcFillInvoker(this, documentDb); } - private boolean legacyEmulationConfigIsSet(DocumentDatabase db) { - LegacyEmulationConfig config = db.getDocsumDefinitionSet().legacyEmulationConfig(); - if (config.forceFillEmptyFields()) - return true; - if (config.stringBackedFeatureData()) - return true; - if (config.stringBackedStructuredData()) - return true; - return false; - } - public Compressor compressor() { return compressor; } diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java deleted file mode 100644 index 1607d108722..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.Collections; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - - -/** - * Superclass for http client searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPClientSearcher extends HTTPClientSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPClientSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id, ConfiguredSearcherHelper.toConnectionList(providerConfig), new HTTPParameters(providerConfig), manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPClientSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Forwards to the next in chain fill(result,summaryName) */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - execution.fill(result,summaryName); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java deleted file mode 100644 index 9f99a790df0..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import java.util.Collections; - - -/** - * Superclass for http provider searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPProviderSearcher extends HTTPProviderSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - } - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,HTTPParameters parameters, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),parameters, manager); - } - - /** - * Override this to provider multi-phase result filling towards a backend. - * This default implementation does nothing. - */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java deleted file mode 100644 index 0fa235709f9..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * Some static helper classes for configured*Searcher classes - * - * @author bratseth - */ -// TODO: Remove on Vespa 7 -class ConfiguredSearcherHelper { - - /** No instantiation */ - private ConfiguredSearcherHelper() { } - - public static List<Connection> toConnectionList(ProviderConfig providerConfig) { - List<Connection> connections=new ArrayList<>(); - for(ProviderConfig.Node node : providerConfig.node()) { - connections.add(new Connection(node.host(), node.port())); - } - return connections; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java b/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java deleted file mode 100644 index 4f624a12547..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Represents a connection to a particular node (host/port). - * Right now this is just a container of connection parameters, but might be extended to - * contain an open connection later. - * The host and port state is immutable. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class Connection { - - private String host; - private int port; - - public Connection(String host,int port) { - this.host=host; - this.port=port; - } - - public String getHost() { return host; } - - public int getPort() { return port; } - - public String toString() { - return "http connection '" + host + ":" + port + "'"; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java deleted file mode 100644 index 0407ddeca1b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import org.apache.http.HttpEntity; -import org.apache.http.entity.HttpEntityWrapper; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.zip.GZIPInputStream; - -/** - * Used by HTTPSearcher when talking to services returning compressed content. - * - * @author Mainak Mandal - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class GzipDecompressingEntity extends HttpEntityWrapper { - - private static class Resources { - - byte [] buffer; - int total; - - Resources() { - total = 0; - buffer = new byte[65536]; - } - void drain(InputStream zipStream) throws IOException { - int numRead = zipStream.read(buffer, total, buffer.length); - while (numRead != -1) { - total += numRead; - if ((total + 65536) > buffer.length) { - buffer = Arrays.copyOf(buffer, buffer.length + numRead); - } - numRead = zipStream.read(buffer, total, buffer.length - total); - } - } - - } - - private final Resources resources = new Resources(); - - public GzipDecompressingEntity(final HttpEntity entity) throws IllegalStateException, IOException { - super(entity); - GZIPInputStream gz = new GZIPInputStream(entity.getContent()); - InputStream zipStream = new BufferedInputStream(gz); - try { - resources.drain(zipStream); - } catch (IOException e) { - throw e; - } finally { - zipStream.close(); - } - } - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - - final ByteBuffer buff = ByteBuffer.wrap(resources.buffer, 0, resources.total); - return new InputStream() { - - @Override - public int available() throws IOException { - return buff.remaining(); - } - - @Override - public int read() throws IOException { - if (buff.hasRemaining()) - return buff.get() & 0xFF; - - return -1; - } - - @Override - public int read(byte[] b) throws IOException { - if (!buff.hasRemaining()) - return -1; - - int len = b.length; - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, 0, len); - return len; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, off, len); - return len; - } - - @Override - public long skip(long n) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (n > buff.remaining()) - n = buff.remaining(); - - buff.position(buff.position() + (int) n); - return n; - } - }; - } - - @Override - public long getContentLength() { - return resources.total; - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - outstream.write(resources.buffer, 0, resources.total); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java deleted file mode 100644 index ee093adc6fc..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -/** - * A utility parent for searchers which gets data from web services which is incorporated into the query. - * This searcher will take care of implementing the search method while the extending class implements - * {@link #getQueryMap} and {@link #handleResponse} to create the http request and handle the response, respectively. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, is this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPClientSearcher extends HTTPSearcher { - - static final CompoundName REQUEST_META_CARRIER = new CompoundName("com.yahoo.search.federation.http.HTTPClientSearcher_requestMeta"); - - protected final static Logger log = Logger.getLogger(HTTPClientSearcher.class.getName()); - - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param path the path portion of the url to be used - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics) { - super(id, connections, path, statistics); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics, - CertificateStore certificateStore) { - super(id, connections, path, statistics, certificateStore); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, Statistics statistics) { - super(id, connections, parameters, statistics); - } - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - * @param certificateStore the certificate store to use to pass certificates in requests - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - } - - /** Overridden to avoid interfering with errors from nested searchers, which is inappropriate for a <i>client</i> */ - @Override - public Result robustSearch(Query query, Execution execution, Connection connection) { - return search(query,execution,connection); - } - - /** Implements a search towards the connection chosen by the cluster searcher for this query */ - @Override - public Result search(Query query, Execution execution, Connection connection) { - Hit requestMeta = doHttpRequest(query, connection); - Result result = execution.search(query); - result.hits().add(requestMeta); - return result; - } - - private Hit doHttpRequest(Query query, Connection connection) { - URI uri; - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - query.properties().set(REQUEST_META_CARRIER, requestMeta); - - query.trace("Created request information hit",false,9); - try { - uri = getURI(query, connection); - } catch (MalformedURLException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } catch (URISyntaxException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } - - HttpEntity entity; - try { - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - entity = getEntity(uri, requestMeta, query); - } catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this + " using " + connection + " for " + - query + ": " + Exceptions.toMessageString(e))); - return requestMeta; - } catch (TimeoutException e) { - query.errors().add(ErrorMessage.createTimeout("HTTP traffic timed out in " - + this + " for " + query + ": " + e.getMessage())); - return requestMeta; - } - if (entity==null) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "No result from connecting to HTTP backend in " + this + " using " + connection + " for " + query)); - return requestMeta; - } - - try { - query = handleResponse(entity,query); - } - catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - } finally { - cleanupHttpEntity(entity); - } - return requestMeta; - } - - /** Overrides to pass the query on to the next searcher */ - @Override - public Result search(Query query, Execution execution, ErrorMessage error) { - query.errors().add(error); - return execution.search(query); - } - - /** Do nothing on fill in client searchers */ - @Override - public void fill(Result result,String summaryClass,Execution execution,Connection connection) { - } - - /** - * Convenience hook for unmarshalling the response and adding the information to the query. - * Implement this or <code>handleResponse(entity,query)</code> in any subclass. - * This default implementation throws an exception. - * - * @param inputStream the stream containing the data from the http service - * @param contentLength the length of the content in the stream in bytes, or a negative number if not known - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - throw new UnsupportedOperationException("handleResponse must be implemented by " + this); - } - - /** - * Unmarshals the response and adds the resulting data to the given query. - * This default implementation calls - * <code>return handleResponse(entity.getContent(), entity.getContentLength(), query);</code> - * (and does some detailed query tracing). - * - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(HttpEntity entity, Query query) throws IOException { - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query = handleResponse(entity.getContent(), len, query); - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - return query; - } - - /** Never retry individual queries to clients for now */ - @Override - protected boolean shouldRetry(Query query,Result result) { return false; } - - /** - * numHits and offset should not be part of the cache key as cache supports - * partial read/write that is only one cache entry is maintained per query - * irrespective of the offset and numhits. - */ - public abstract Map<String, String> getCacheKey(Query q); - - /** - * Adds all key-values starting by "service." + getClientName() in query.properties(). - * Returns the empty map if {@link #getServiceName} is not overridden. - */ - @Override - public Map<String,String> getQueryMap(Query query) { - LinkedHashMap<String, String> queryMap=new LinkedHashMap<>(); - if (getServiceName().isEmpty()) return queryMap; - - for (Map.Entry<String,Object> objectProperty : query.properties().listProperties("service." + getServiceName()).entrySet()) // TODO: Make more efficient using CompoundName - queryMap.put(objectProperty.getKey(),objectProperty.getValue().toString()); - return queryMap; - } - - /** - * Override this to return the name of the service this is a client of. - * This is used to look up service specific properties as service.getServiceName.serviceSpecificProperty. - * This default implementation returns "", which means service specific parameters will not be used. - */ - protected String getServiceName() { return ""; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java deleted file mode 100644 index a7b32678c1a..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.base.Preconditions; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnPerRouteBean; -import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.params.HttpParams; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * A set of parameters for talking to an http backend - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public final class HTTPParameters { - - public static final String RETRIES = "com.yahoo.search.federation.http.retries"; - - private boolean frozen=false; - - // All timing parameters below are in milliseconds - /** The url request path portion */ - private String path="/"; - private int connectionTimeout=2000; - private int readTimeout=5000; - private boolean persistentConnections=true; - private boolean enableProxy = false; - private String proxyHost = "localhost"; - private int proxyPort = 1080; - private String method = "GET"; - private String schema = "http"; - private String inputEncoding = "utf-8"; - private String outputEncoding = "utf-8"; - private int maxTotalConnections=10000; - private int maxConnectionsPerRoute=10000; - private int socketBufferSizeBytes=-1; - private int retries = 1; - private int configuredReadTimeout = -1; - private int configuredConnectionTimeout = -1; - private int connectionPoolTimeout = -1; - private String certificateProxy = null; - private int certificatePort = 0; - private String certificateApplicationId = null; - private boolean certificateUseProxy = false; - private long certificateTtl = 0L; - private long certificateRetry = 0L; - - private PingOption.Enum pingOption = PingOption.NORMAL; - - - private boolean followRedirects = true; - - public HTTPParameters() {} - - public HTTPParameters(String path) { - setPath(path); - } - - public HTTPParameters(ProviderConfig providerConfig) { - configuredReadTimeout = (int) (providerConfig.readTimeout() * 1000.0d); - configuredConnectionTimeout = (int) (providerConfig.connectionTimeout() * 1000.0d); - connectionPoolTimeout = (int) (providerConfig.connectionPoolTimeout() * 1000.0d); - retries = providerConfig.retries(); - setPath(providerConfig.path()); - certificateUseProxy = providerConfig.yca().useProxy(); - if (certificateUseProxy) { - certificateProxy = providerConfig.yca().host(); - certificatePort = providerConfig.yca().port(); - } - certificateApplicationId = providerConfig.yca().applicationId(); - certificateTtl = providerConfig.yca().ttl() * 1000L; - certificateRetry = providerConfig.yca().retry() * 1000L; - followRedirects = providerConfig.followRedirects(); - pingOption = providerConfig.pingOption(); - } - - /** - * Set the url path to use in queries to this. If the argument is null or empty the path is set to "/". - * If a leading "/" is missing, it is added automatically. - */ - public final void setPath(String path) { - if (path==null || path.isEmpty()) path="/"; - - if (! path.startsWith("/")) - path="/" + path; - this.path = path; - } - - public PingOption.Enum getPingOption() { - return pingOption; - } - - public void setPingOption(PingOption.Enum pingOption) { - Preconditions.checkNotNull(pingOption); - ensureNotFrozen(); - this.pingOption = pingOption; - } - - /** Returns the url path. Default is "/". */ - public String getPath() { return path; } - - public boolean getFollowRedirects() { - return followRedirects; - } - - public void setFollowRedirects(boolean followRedirects) { - ensureNotFrozen(); - this.followRedirects = followRedirects; - } - - - public void setConnectionTimeout(int connectionTimeout) { - ensureNotFrozen(); - this.connectionTimeout=connectionTimeout; - } - - /** Returns the connection timeout in milliseconds. Default is 2000. */ - public int getConnectionTimeout() { return connectionTimeout; } - - public void setReadTimeout(int readTimeout) { - ensureNotFrozen(); - this.readTimeout=readTimeout; - } - - /** Returns the read timeout in milliseconds. Default is 5000. */ - public int getReadTimeout() { return readTimeout; } - - /** - * <b>Note: This is currently largely a noop: Connections are reused even when this is set to true. - * The setting will change from sharing connections between threads to only reusing it within a thread - * but it is still reused.</b> - */ - public void setPersistentConnections(boolean persistentConnections) { - ensureNotFrozen(); - this.persistentConnections=persistentConnections; - } - - /** Returns whether this should use persistent connections. Default is true. */ - public boolean getPersistentConnections() { return persistentConnections; } - - /** Returns whether proxying should be enabled. Default is false. */ - public boolean getEnableProxy() { return enableProxy; } - - public void setEnableProxy(boolean enableProxy ) { - ensureNotFrozen(); - this.enableProxy=enableProxy; - } - - /** Returns the proxy type to use (if enabled). Default is "http". */ - public String getProxyType() { - return "http"; - } - - public void setProxyHost(String proxyHost) { - ensureNotFrozen(); - this.proxyHost=proxyHost; - } - - /** Returns the proxy host to use (if enabled). Default is "localhost". */ - public String getProxyHost() { return proxyHost; } - - public void setProxyPort(int proxyPort) { - ensureNotFrozen(); - this.proxyPort=proxyPort; - } - - /** Returns the proxy port to use (if enabled). Default is 1080. */ - public int getProxyPort() { return proxyPort; } - - public void setMethod(String method) { - ensureNotFrozen(); - this.method=method; - } - - /** Returns the http method to use. Default is "GET". */ - public String getMethod() { return method; } - - public void setSchema(String schema) { - ensureNotFrozen(); - this.schema=schema; - } - - /** Returns the schema to use. Default is "http". */ - public String getSchema() { return schema; } - - public void setInputEncoding(String inputEncoding) { - ensureNotFrozen(); - this.inputEncoding=inputEncoding; - } - - /** Returns the input encoding. Default is "utf-8". */ - public String getInputEncoding() { return inputEncoding; } - - public void setOutputEncoding(String outputEncoding) { - ensureNotFrozen(); - this.outputEncoding=outputEncoding; - } - - /** Returns the output encoding. Default is "utf-8". */ - public String getOutputEncoding() { return outputEncoding; } - - /** Make this unmodifiable. Note that any thread synchronization must be done outside this object. */ - public void freeze() { - frozen=true; - } - - private void ensureNotFrozen() { - if (frozen) throw new IllegalStateException("Cannot modify frozen " + this); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams() { - return toHttpParams(connectionTimeout, readTimeout); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams(int connectionTimeout, int readTimeout) { - HttpParams params = new BasicHttpParams(); - // force use of configured value if available - if (configuredConnectionTimeout > 0) { - HttpConnectionParams.setConnectionTimeout(params, configuredConnectionTimeout); - } else { - HttpConnectionParams.setConnectionTimeout(params, connectionTimeout); - } - if (configuredReadTimeout > 0) { - HttpConnectionParams.setSoTimeout(params, configuredReadTimeout); - } else { - HttpConnectionParams.setSoTimeout(params, readTimeout); - } - if (socketBufferSizeBytes > 0) { - HttpConnectionParams.setSocketBufferSize(params, socketBufferSizeBytes); - } - if (connectionPoolTimeout > 0) { - ConnManagerParams.setTimeout(params, connectionPoolTimeout); - } - ConnManagerParams.setMaxTotalConnections(params, maxTotalConnections); - ConnManagerParams.setMaxConnectionsPerRoute(params, new ConnPerRouteBean(maxConnectionsPerRoute)); - if (retries >= 0) { - params.setIntParameter(RETRIES, retries); - } - params.setParameter("http.protocol.handle-redirects", followRedirects); - return params; - } - - public int getMaxTotalConnections() { - return maxTotalConnections; - } - - public void setMaxTotalConnections(int maxTotalConnections) { - ensureNotFrozen(); - this.maxTotalConnections = maxTotalConnections; - } - - public int getMaxConnectionsPerRoute() { - return maxConnectionsPerRoute; - } - - public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) { - ensureNotFrozen(); - this.maxConnectionsPerRoute = maxConnectionsPerRoute; - } - - public int getSocketBufferSizeBytes() { - return socketBufferSizeBytes; - } - - public void setSocketBufferSizeBytes(int socketBufferSizeBytes) { - ensureNotFrozen(); - this.socketBufferSizeBytes = socketBufferSizeBytes; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - ensureNotFrozen(); - this.retries = retries; - } - - public String getYcaProxy() { - return certificateProxy; - } - - public int getYcaPort() { - return certificatePort; - } - - public String getYcaApplicationId() { - return certificateApplicationId; - } - - public boolean getYcaUseProxy() { - return certificateUseProxy; - } - - public long getYcaTtl() { - return certificateTtl; - } - - public long getYcaRetry() { - return certificateRetry; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java deleted file mode 100644 index c72c2f26a1c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.collect.ImmutableList; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.federation.FederationSearcher; -import com.yahoo.search.query.Properties; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.statistics.Value; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.*; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Superclass of searchers which talks to HTTP backends. Implement a subclass to talk to a backend - * over HTTP which is not supported by the platform out of the box. - * <p> - * Implementations must override one of the <code>unmarshal</code> methods to unmarshal the response. - * </p> - * - * @author Arne Bergene Fossaa - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPProviderSearcher extends HTTPSearcher { - - private final Counter emptyResults; - private final Value hitsPerQuery; - private final Value responseLatency; - private final Counter readTimeouts; - - private final static List<String> excludedSourceProperties = ImmutableList.of("offset", "hits", "provider"); - - protected final static Logger log = Logger.getLogger(HTTPProviderSearcher.class.getName()); - - /** The name of the cache used (which is just getid().stringValue(), or null if no cache is used */ - protected String cacheName=null; - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections,String path, Statistics statistics) { - this(id,connections,new HTTPParameters(path), statistics); - } - - /** Creates a http provider searcher using id.getName as provider name */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, String path, - Statistics statistics, CertificateStore certificateStore) { - this(id, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(id, connections, parameters, statistics, new ThrowingCertificateStore()); - } - - /** - * Creates a provider searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - String suffix = "_" + getId().getName().replace('.', '_'); - hitsPerQuery = new Value("hits_per_query" + suffix, statistics, - new Value.Parameters().setLogRaw(false).setNameExtension(false).setLogMean(true)); - responseLatency = new Value(LOG_LATENCY_START + suffix, statistics, - new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false)); - emptyResults = new Counter("empty_results" + suffix, statistics, false); - readTimeouts = new Counter(LOG_READ_TIMEOUT_PREFIX + suffix, statistics, false); - } - - /** @deprecated this method does nothing */ - @Deprecated // OK - protected void configureCache(final QrBinaryCacheConfig cacheConfig,final QrBinaryCacheRegionConfig regionConfig) { - } - - /** - * Unmarshal the stream by converting it to hits and adding the hits to the given result. - * A convenience hook called by the default <code>unmarshal(entity,result).</code> - * Override this in subclasses which does not override <code>unmarshal(entity,result).</code> - * <p> - * This default implementation throws an exception. - * - * @param stream the stream of data returned - * @param contentLength the length of the content in bytes if known, or a negative number if unknown - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { - throw new UnsupportedOperationException("Unmarshal must be implemented by " + this); - } - - /** - * Unmarshal the result from an http entity. This default implementation calls - * <code>unmarshal(entity.getContent(), entity.getContentLength(), result)</code> - * (and does some detailed query tracing). - * - * @param entity the entity containing the data to unmarshal - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(HttpEntity entity,Result result) throws IOException { - Query query=result.getQuery(); - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query.trace("Unmarshaling result.", false, 6); - unmarshal(entity.getContent(), len, result); - - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - - } - - protected void addNonExcludedSourceProperties(Query query, Map<String, String> queryMap) { - Properties sourceProperties = FederationSearcher.getSourceProperties(query); - if (sourceProperties != null) { - for(Map.Entry<String, Object> entry : sourceProperties.listProperties("").entrySet()) { - if (!excludedSourceProperties.contains(entry.getKey())) { - queryMap.put(entry.getKey(), entry.getValue().toString()); - } - } - } - } - - /** - * Hook called at the moment the result is returned from this searcher. This default implementation - * does <code>return result</code>. - * - * @param result the result which is to be returned - * @param requestMeta the request information hit, or null if none was created (e.g if this was a cache lookup) - * @param e the exception caused during execution of this query, or null if none - * @return the result which is returned upwards - */ - protected Result inspectAndReturnFinalResult(Result result, Hit requestMeta, Exception e) { - return result; - } - - private Result statisticsBeforeInspection(Result result, Hit requestMeta, Exception e) { - int hitCount = result.getConcreteHitCount(); - if (hitCount == 0) { - emptyResults.increment(); - } - hitsPerQuery.put((double) hitCount); - - if (requestMeta != null) { - requestMeta.setField(LOG_HITCOUNT, Integer.valueOf(hitCount)); - } - - return inspectAndReturnFinalResult(result, requestMeta, e); - } - - - @Override - protected void logResponseLatency(long latency) { - responseLatency.put((double) latency); - } - - @Override - public Result search(Query query, Execution execution,Connection connection) { - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - Result result = new Result(query); - result.hits().add(requestMeta); - query.trace("Created request information hit", false, 9); - - try { - URI uri = getURI(query, requestMeta, connection); - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - long requestStartTime = System.currentTimeMillis(); - - HttpEntity entity = getEntity(uri, requestMeta, query); - - // Why should consumeEntity call inspectAndReturnFinalResult itself? - // Seems confusing to me. - return entity == null - ? statisticsBeforeInspection(result, requestMeta, null) - : consumeEntity(entity, query, result, requestMeta, requestStartTime); - - } catch (MalformedURLException|URISyntaxException e) { - result.hits().addError(createMalformedUrlError(query,e)); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (TimeoutException e) { - result.hits().addError(ErrorMessage.createTimeout("No time left for HTTP traffic in " - + this - + " for " + query + ": " + e.getMessage())); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this - + " for " + query + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, requestMeta, e); - } - } - - private Result consumeEntity(HttpEntity entity, Query query, Result result, Hit logHit, long requestStartTime) { - - try { - // remove some time from timeout to allow for close calls with return result - unmarshal(new TimedHttpEntity(entity, query.getStartTime(), Math.max(1, query.getTimeout() - 10)), result); - logHit.setField(LOG_LATENCY_FINISH, System.currentTimeMillis() - requestStartTime); - return statisticsBeforeInspection(result, logHit, null); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, logHit, e); - } catch (TimeoutException e) { - readTimeouts.increment(); - result.hits().addError(ErrorMessage - .createTimeout("Timed out while reading/unmarshaling from backend in " - + this + " for " + query - + ": " + e.getMessage())); - return statisticsBeforeInspection(result, logHit, e); - } finally { // TODO: The scope of this finally must be enlarged to release the connection also on errors - cleanupHttpEntity(entity); - } - } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns the query.properties() prefixed by - * "source.[sourceName]" or "property.[propertyName]" - * (by calling {@link #addNonExcludedSourceProperties}). - */ - @Override - public Map<String,String> getQueryMap(Query query) { - Map<String,String> queryMap = super.getQueryMap(query); - addNonExcludedSourceProperties(query, queryMap); - return queryMap; - } - - /** - * @deprecated the cache key is ignored as there is no built-in caching support - */ - @Deprecated // OK - public abstract Map<String, String> getCacheKey(Query q); - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java deleted file mode 100644 index edf347bd84e..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java +++ /dev/null @@ -1,961 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.inject.Inject; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.cluster.ClusterSearcher; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; - -import org.apache.http.*; -import org.apache.http.client.HttpClient; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnRoutePNames; -import org.apache.http.conn.routing.HttpRoutePlanner; -import org.apache.http.conn.scheme.PlainSocketFactory; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.scheme.SchemeRegistry; -import org.apache.http.conn.ssl.SSLSocketFactory; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.DefaultHttpRoutePlanner; -import org.apache.http.impl.conn.SingleClientConnManager; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.HttpParams; -import org.apache.http.params.HttpProtocolParams; -import org.apache.http.protocol.BasicHttpContext; -import org.apache.http.protocol.ExecutionContext; -import org.apache.http.protocol.HttpContext; -import org.apache.http.protocol.HttpRequestExecutor; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.SSLHandshakeException; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.UnsupportedEncodingException; -import java.net.*; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Generic superclass of searchers making connections to some HTTP service. This - * supports clustered connections - a list of alternative servers may be given, - * requests will be hashed across these and failed over in case some are down. - * <p> - * This simply provides some utility methods for working with http connections - * and implements ping against the service. - * - * <p>This searcher contains code from the Apache httpcomponents client library, - * licensed to the Apache Software Foundation under the Apache License, Version - * 2.0. Please refer to http://www.apache.org/licenses/LICENSE-2.0 for details. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, as this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_HITCOUNT - * <dd> - * Number of concrete hits in the result returned by this provider. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - * - * @author Arne Bergene Fossaa - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPSearcher extends ClusterSearcher<Connection> { - - protected static final String YCA_HTTP_HEADER = "Yahoo-App-Auth"; - - private static final Charset iso8859Charset = Charset.forName("ISO-8859-1"); - - // Logging field name constants - public static final String LOG_PATH = "path"; - public static final String LOG_PORT = "port"; - public static final String LOG_HOST = "host"; - public static final String LOG_IP_ADDRESS = "ip_address"; - public static final String IP_ADDRESS_UNKNOWN = "unknown"; - - public static final String LOG_SCHEME = "scheme"; - public static final String LOG_URI = "uri"; - public static final String LOG_PROXY_PORT = "proxy_port"; - public static final String LOG_PROXY_HOST = "proxy_host"; - public static final String LOG_PROXY_TYPE = "proxy_type"; - public static final String LOG_STATUS = "status"; - public static final String LOG_LATENCY_FINISH = "latency_finish"; - public static final String LOG_LATENCY_START = "latency_start"; - public static final String LOG_LATENCY_CONNECT = "latency_connect"; - public static final String LOG_QUERY_PARAM_PREFIX = "query_param_"; - public static final String LOG_HEADER_PREFIX = "header_"; - public static final String LOG_RESPONSE_HEADER_PREFIX = "response_header_"; - public static final String LOG_HITCOUNT = "hit_count"; - public static final String LOG_CONNECT_TIMEOUT_PREFIX = "connect_timeout_"; - public static final String LOG_READ_TIMEOUT_PREFIX = "read_timeout_"; - - protected final Logger log = Logger.getLogger(HTTPSearcher.class.getName()); - - /** The HTTP parameters to use. Assigned in the constructor */ - private HTTPParameters httpParameters; - - private final Counter connectTimeouts; - - /** Whether to use certificates */ - protected boolean useCertificate = false; - - private final CertificateStore certificateStore; - - /** The (optional) certificate application ID. */ - private String certificateApplicationId = null; - - /** The (optional) certificate server proxy */ - protected HttpHost certificateProxy = null; - - /** Certificate cache TTL in ms */ - private long certificateTtl = 0L; - - /** Certificate server retry rate in the cache if no cert is found, in ms */ - private long certificateRetry = 0L; - - /** Set at construction if this is using persistent connections */ - private ClientConnectionManager sharedConnectionManager = null; - - /** Set at construction if using non-persistent connections */ - private ThreadLocal<SingleClientConnManager> singleClientConnManagerThreadLocal = null; - - private static final SchemeRegistry schemeRegistry = new SchemeRegistry(); - - static { - schemeRegistry.register(new Scheme("http", PlainSocketFactory - .getSocketFactory(), 80)); - schemeRegistry.register(new Scheme("https", SSLSocketFactory - .getSocketFactory(), 443)); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics) { - this(componentId, connections, new HTTPParameters(path), statistics, new ThrowingCertificateStore()); - } - - /** Creates a http searcher with default connection and read timeouts (currently 2 and 5s respectively) */ - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics, - CertificateStore certificateStore) { - this(componentId, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(componentId, connections, parameters, statistics, new ThrowingCertificateStore()); - } - /** - * Creates a http searcher - * - * @param componentId the id of this instance - * @param connections the connections to establish to the backend nodes - * @param parameters the http parameters to use. This object will be frozen if it isn't already - */ - @Inject - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(componentId,connections,false); - String suffix = "_" + getId().getName().replace('.', '_'); - - connectTimeouts = new Counter(LOG_CONNECT_TIMEOUT_PREFIX + suffix, statistics, false); - - parameters.freeze(); - this.httpParameters = parameters; - this.certificateStore = certificateStore; - - if (parameters.getPersistentConnections()) { - HttpParams params=parameters.toHttpParams(); - HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); - ConnManagerParams.setTimeout(params, 10); - sharedConnectionManager = new ThreadSafeClientConnManager(params, schemeRegistry); - Thread connectionPurgerThread = new Thread(() -> { - //this is the default value in yahoo jvm installations - long DNSTTLSec = 120; - while (true) { - try { - Thread.sleep(DNSTTLSec * 1000); - if (sharedConnectionManager == null) - continue; - - sharedConnectionManager.closeExpiredConnections(); - DNSTTLSec = Long.valueOf(java.security.Security - .getProperty("networkaddress.cache.ttl")); - //No DNS TTL, no need to close idle connections - if (DNSTTLSec <= 0) { - DNSTTLSec = 120; - continue; - } - sharedConnectionManager.closeIdleConnections(2 * DNSTTLSec, TimeUnit.SECONDS); - } catch (InterruptedException e) { - return; - } catch (NumberFormatException e) { - continue; - } - } - }); - connectionPurgerThread.setDaemon(true); - connectionPurgerThread.start(); - - } - else { - singleClientConnManagerThreadLocal =new ThreadLocal<>(); - } - - initializeCertificate(httpParameters, certificateStore); - } - - /** - * Initialize certificate store and proxy if they have been set to non-null, - * non-empty values. It will wrap thrown exceptions from the certificate store into - * RuntimeException and propagate them. - */ - private void initializeCertificate(HTTPParameters parameters, CertificateStore certificateStore) { - String applicationId = parameters.getYcaApplicationId(); - String proxy = parameters.getYcaProxy(); - int port = parameters.getYcaPort(); - long ttl = parameters.getYcaTtl(); - long retry = parameters.getYcaRetry(); - - if (applicationId != null && !applicationId.trim().isEmpty()) { - initializeCertificate(applicationId, ttl, retry, certificateStore); - } - - if (parameters.getYcaUseProxy()) { - initializeProxy(proxy, port); - } - } - - /** Returns the HTTP parameters used in this. This is always frozen */ - public HTTPParameters getParameters() { return httpParameters; } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns an empty LinkedHashMap. - */ - public Map<String,String> getQueryMap(Query query) { - return new LinkedHashMap<>(); - } - - /** - * Initialize the certificate. - * This will warn but not throw if certificates could not be loaded, as the certificates - * are external state which can fail independently. - */ - private void initializeCertificate(String applicationId, long ttl, long retry, CertificateStore certificateStore) { - try { - // get the certificate, i.e. init the cache and check integrity - String certificate = certificateStore.getCertificate(applicationId, ttl, retry); - if (certificate == null) { - getLogger().log(LogLevel.WARNING, "No certificate found for application '" + applicationId + "'"); - return; - } - - this.useCertificate = true; - this.certificateApplicationId = applicationId; - this.certificateTtl = ttl; - this.certificateRetry = retry; - getLogger().log(LogLevel.CONFIG, "Got certificate: " + certificate); - } - catch (Exception e) { - getLogger().log(LogLevel.WARNING,"Exception while initializing certificate for application '" + - applicationId + "' in " + this, e); - } - } - - /** - * Initialize the certificate proxy setting. - */ - private void initializeProxy(String host, int port) { - certificateProxy = new HttpHost(host, port); - getLogger().log(LogLevel.CONFIG, "Proxy is configured; will use proxy: " + certificateProxy); - } - - /** - * Same a {@code getURI(query, offset, hits, null)}. - * @see #getURI(Query, Hit, Connection) - */ - protected URI getURI(Query query,Connection connection) throws MalformedURLException, URISyntaxException { - Hit requestMeta; - try { - requestMeta = (Hit) query.properties().get(HTTPClientSearcher.REQUEST_META_CARRIER); - } catch (ClassCastException e) { - requestMeta = null; - } - return getURI(query, requestMeta, connection); - } - - /** - * Creates the URI for a query. - * Populates the {@code requestMeta} meta hit with the created URI HTTP properties. - * - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - */ - protected URI getURI(Query query, Hit requestMeta, Connection connection) - throws MalformedURLException, URISyntaxException { - StringBuilder parameters = new StringBuilder(); - - Map<String, String> queries = getQueryMap(query); - if (queries.size() > 0) { - Iterator<Map.Entry<String, String>> mapIterator = queries.entrySet().iterator(); - parameters.append("?"); - try { - Map.Entry<String, String> entry; - while (mapIterator.hasNext()) { - entry = mapIterator.next(); - - if (requestMeta != null) - requestMeta.setField(LOG_QUERY_PARAM_PREFIX - + entry.getKey(), entry.getValue()); - - parameters.append(entry.getKey() + "=" + URLEncoder.encode(entry.getValue(), - httpParameters.getInputEncoding())); - if (mapIterator.hasNext()) { - parameters.append("&"); - } - } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Unknown input encoding set in " + this, e); - } - } - - URI uri = new URL(httpParameters.getSchema(), connection.getHost(), - connection.getPort(), getPath() + parameters.toString()).toURI(); - if (requestMeta != null) { - requestMeta.setField(LOG_URI, uri.toString()); - requestMeta.setField(LOG_SCHEME, uri.getScheme()); - requestMeta.setField(LOG_HOST, uri.getHost()); - requestMeta.setField(LOG_PORT, uri.getPort()); - requestMeta.setField(LOG_PATH, uri.getPath()); - } - return uri; - } - - /** - * Called by getURI() to get the path of the URI for the external service. - * The default implementation returns httpParameters.getPath(); subclasses - * which only wants to override the path from httpParameters may use this - * method instead of overriding all of getURI(). - * - * @return the path to use for getURI - */ - protected String getPath() { - return httpParameters.getPath(); - } - - /** - * The URI that is used to check if the provider is up or down. This will again be used in the - * checkPing method by checking that we get a response that has a good status code (below 300). If better - * validation than just status code checking is needed, override the checkPing method. - */ - protected URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException { - return new URL(httpParameters.getSchema(),connection.getHost(),connection.getPort(),getPingPath()).toURI(); - } - - /** - * Called by getPingURI() to get the path of the URI for pinging the - * external service. The default implementation returns - * httpParameters.getPath(); subclasses which only wants to override the - * path from httpParameters may use this method instead of overriding all of - * getPingURI(). - * - * @return the path to use for getPingURI - */ - protected String getPingPath() { - return httpParameters.getPath(); - } - - /** - * Checks if the response is valid. - * @param response The response from the ping request - * @param pong The pong result to return back to the calling method. This method - * will add an error to the pong result (using addError) if the status of the HTTP response is 300 or above. - */ - protected void checkPing(HttpResponse response, Pong pong) { - if (response.getStatusLine().getStatusCode() >= 300) { - pong.addError(com.yahoo.search.result.ErrorMessage.createBackendCommunicationError( - "Got error " + response.getStatusLine().getStatusCode() - + " when contacting backend") - ); - } - } - - /** - * Pinging in HTTPBackend is done by creating a PING uri from http://host:port/path. - * If this returns a status that is below 300, the ping is considered good. - * - * If another uri is needed for pinging, reimplement getPingURI. - * - * Override either this method to change how ping - */ - @Override - public Pong ping(Ping ping, Connection connection) { - URI uri = null; - Pong pong = new Pong(); - HttpResponse response = null; - - if (httpParameters.getPingOption() == PingOption.DISABLE) - return pong; - - try { - uri = getPingURI(connection); - if (uri == null) - pong.addError(ErrorMessage.createIllegalQuery("Ping uri is null")); - if (uri.getHost()==null) { - pong.addError(ErrorMessage.createIllegalQuery("Ping uri has no host")); - uri=null; - } - } catch (MalformedURLException | URISyntaxException e) { - pong.addError(ErrorMessage.createIllegalQuery("Malformed ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + - " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } - - if (uri == null) return pong; - pong.setPingInfo("using uri '" + uri + "'"); - - try { - response = getPingResponse(uri, ping); - checkPing(response, pong); - } catch (IOException e) { - // We do not have a valid ping - pong.addError(ErrorMessage.createBackendCommunicationError( - "Exception thrown when pinging with url '" + uri + "': " + Exceptions.toMessageString(e))); - } catch (TimeoutException e) { - pong.addError(ErrorMessage.createTimeout("Timeout for ping " + uri + " in " + this + ": " + e.getMessage())); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } finally { - if (response != null) { - cleanupHttpEntity(response.getEntity()); - } - } - - return pong; - } - - private HttpResponse getPingResponse(URI uri, Ping ping) throws IOException { - long timeLeft = ping.getTimeout(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - - Map<String, String> requestHeaders = null; - if (httpParameters.getPingOption() == PingOption.YCA) - requestHeaders = generateYCAHeaders(); - - return getResponse(uri, null, requestHeaders, null, connectionTimeout, readTimeout); - } - - /** - * Same a {@code getEntity(uri, null)}. - * @param uri resource to fetch - * @param query the originating query - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Query query) throws IOException{ - return getEntity(uri, null, query); - } - - - /** - * Gets the HTTP entity that holds the response contents. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query - * @return the http entity, or null if none - * @throws java.io.IOException Whenever HTTP status code is in the 300 or higher range. - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { - if (query.getTimeLeft() <= 0) { - throw new TimeoutException("No time left for querying external backend."); - } - HttpResponse response = getResponse(uri, requestMeta, query); - StatusLine statusLine = response.getStatusLine(); - - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_STATUS, statusLine.getStatusCode()); - for (HeaderIterator headers = response.headerIterator(); headers.hasNext(); ) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_RESPONSE_HEADER_PREFIX + h.getName(), h.getValue()); - } - } - - if (statusLine.getStatusCode() >= 300) { - HttpEntity entity = response.getEntity(); - String message = createServerReporterErrorMessage(statusLine, entity); - cleanupHttpEntity(response.getEntity()); - throw new IOException(message); - } - - return response.getEntity(); - } - - private String createServerReporterErrorMessage(StatusLine statusLine, HttpEntity entity) { - String message = "Error when trying to connect to HTTP backend: " - + statusLine.getStatusCode() + " : " + statusLine.getReasonPhrase(); - - try { - if (entity != null) { - message += "(Message = " + EntityUtils.toString(entity) + ")"; - } - } catch (Exception e) { - log.log(LogLevel.WARNING, "Could not get message.", e); - } - - return message; - } - - /** - * Creates a meta hit dedicated to holding logging information. This hit has - * the 'logging:[searcher's ID]' type. - */ - protected Hit createRequestMeta() { - Hit requestMeta = new Hit("logging:" + getId().toString()); - requestMeta.setMeta(true); - requestMeta.types().add("logging"); - return requestMeta; - } - - protected void cleanupHttpEntity(HttpEntity entity) { - if (entity == null) return; - - try { - entity.consumeContent(); - } catch (IOException e) { - // It is ok if do not consume it, the resource will be freed after - // timeout. - // But log it just in case. - log.log(LogLevel.getVespaLogLevel(LogLevel.DEBUG), - "Not able to consume after processing: " + Exceptions.toMessageString(e)); - } - } - - /** - * Same as {@code getResponse(uri, null)}. - */ - protected HttpResponse getResponse(URI uri, Query query) throws IOException{ - return getResponse(uri, null, query); - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query, used to calculate timeouts - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, Query query) throws IOException { - long timeLeft = query.getTimeLeft(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - connectionTimeout = connectionTimeout <= 0 ? 1 : connectionTimeout; - readTimeout = readTimeout <= 0 ? 1 : readTimeout; - HttpEntity reqEntity = getRequestEntity(query, requestMeta); - Map<String, String> reqHeaders = getRequestHeaders(query, requestMeta); - if ((reqEntity == null) && (reqHeaders == null)) { - return getResponse(uri, requestMeta, connectionTimeout, readTimeout); - } else { - return getResponse(uri, reqEntity, reqHeaders, requestMeta, connectionTimeout, readTimeout); - } - } - - /** - * Returns the set of headers to be passed in the http request to provider backend. The default - * implementation returns null, unless certificates are in use. If certificates are used, it will return a map - * only containing the needed certificate headers. - */ - protected Map<String, String> getRequestHeaders(Query query, Hit requestMeta) { - if (useCertificate) { - return generateYCAHeaders(); - } - return null; - } - - /** - * Returns the HTTP request entity to use when making the request for this query. - * This default implementation returns null. - * - * <p> Do return a repeatable entity if HTTP retry is active. - * - * @return the http request entity to use, or null to use the default entity - */ - protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { - return null; - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, int connectionTimeout, int readTimeout) - throws IOException { - return getResponse(uri, null, null, requestMeta, connectionTimeout, readTimeout); - } - - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, HttpEntity reqEntity, - Map<String, String> reqHeaders, Hit requestMeta, - int connectionTimeout, int readTimeout) throws IOException { - - HttpParams httpParams = httpParameters.toHttpParams(connectionTimeout, readTimeout); - HttpClient httpClient = createClient(httpParams); - long start = 0L; - HttpUriRequest request; - if (httpParameters.getEnableProxy() && "http".equals(httpParameters.getProxyType())) { - HttpHost proxy = new HttpHost(httpParameters.getProxyHost(), - httpParameters.getProxyPort(), httpParameters.getProxyType()); - httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_PROXY_TYPE, httpParameters.getProxyType()); - requestMeta.setField(LOG_PROXY_HOST, httpParameters.getProxyHost()); - requestMeta.setField(LOG_PROXY_PORT, httpParameters.getProxyPort()); - } - } - if (reqEntity == null) { - request = createRequest(httpParameters.getMethod(), uri); - } else { - request = createRequest(httpParameters.getMethod(), uri, reqEntity); - } - - if (reqHeaders != null) { - for (Entry<String, String> entry : reqHeaders.entrySet()) { - if (entry.getValue() == null || isAscii(entry.getValue())) { - request.addHeader(entry.getKey(), entry.getValue()); - } else { - byte[] asBytes = Utf8.toBytes(entry.getValue()); - String asLyingString = new String(asBytes, 0, asBytes.length, iso8859Charset); - request.addHeader(entry.getKey(), asLyingString); - } - } - } - - // Logging - if (requestMeta != null) { - for (HeaderIterator headers = request.headerIterator(); headers.hasNext();) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_HEADER_PREFIX + h.getName(), h.getValue()); - } - start = System.currentTimeMillis(); - } - - HttpResponse response; - - try { - HttpContext context = new BasicHttpContext(); - response = httpClient.execute(request, context); - - if (requestMeta != null) { - requestMeta.setField(LOG_IP_ADDRESS, getIpAddress(context)); - } - } catch (ConnectTimeoutException e) { - connectTimeouts.increment(); - throw e; - } - - // Logging - long latencyStart = System.currentTimeMillis() - start; - if (requestMeta != null) { - requestMeta.setField(LOG_LATENCY_START, latencyStart); - } - logResponseLatency(latencyStart); - return response; - } - - private String getIpAddress(HttpContext context) { - HttpConnection connection = (HttpConnection) context.getAttribute(ExecutionContext.HTTP_CONNECTION); - if (connection instanceof HttpInetConnection) { - InetAddress address = ((HttpInetConnection) connection).getRemoteAddress(); - String hostAddress = address.getHostAddress(); - return hostAddress == null ? - IP_ADDRESS_UNKNOWN: - hostAddress; - } else { - getLogger().log(LogLevel.DEBUG, "Unexpected connection type: " + connection.getClass().getName()); - return IP_ADDRESS_UNKNOWN; - } - } - - private boolean isAscii(String value) { - char[] scanBuffer = new char[value.length()]; - value.getChars(0, value.length(), scanBuffer, 0); - for (char c: scanBuffer) - if (c > 127) return false; - return true; - } - - protected void logResponseLatency(long latency) { } - - /** - * Creates a http client for one request. Override to customize the client - * to use, e.g for testing. This default implementation will add a certificate store - * proxy to params if is necessary, and then do - * <code>return new SearcherHttpClient(getConnectionManager(params), params);</code> - */ - protected HttpClient createClient(HttpParams params) { - if (certificateProxy != null) { - params.setParameter(ConnRoutePNames.DEFAULT_PROXY, certificateProxy); - } - return new SearcherHttpClient(getConnectionManager(params), params); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri) { - return createRequest(method, uri, null); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri, HttpEntity entity) { - return new SearcherHttpRequest(method,uri); - } - - /** Get a connection manager which may be used safely from this thread */ - protected ClientConnectionManager getConnectionManager(HttpParams params) { - if (sharedConnectionManager != null) {// We are using shared connections - return sharedConnectionManager; - } else { - SingleClientConnManager singleClientConnManager = singleClientConnManagerThreadLocal.get(); - if (singleClientConnManager == null) { - singleClientConnManager = new SingleClientConnManager(params, schemeRegistry); - singleClientConnManagerThreadLocal.set(singleClientConnManager); - } - return singleClientConnManager; - } - } - - /** Utility method for creating error messages when a url is incorrect */ - protected ErrorMessage createMalformedUrlError(Query query,Exception e) { - return ErrorMessage.createErrorInPluginSearcher("Malformed url in " + this + " for " + query + - ": " + Exceptions.toMessageString(e)); - } - - private Map<String, String> generateYCAHeaders() { - Map<String, String> headers = new HashMap<>(); - String certificate = certificateStore.getCertificate(certificateApplicationId, certificateTtl, certificateRetry); - headers.put(YCA_HTTP_HEADER, certificate); - return headers; - } - - protected static class SearcherHttpClient extends DefaultHttpClient { - - private final int retries; - - public SearcherHttpClient(final ClientConnectionManager conman, final HttpParams params) { - super(conman, params); - retries = params.getIntParameter(HTTPParameters.RETRIES, 1); - addRequestInterceptor((request, context) -> { - if (!request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip"); - } - }); - addResponseInterceptor((response, context) -> { - HttpEntity entity = response.getEntity(); - if (entity == null) return; - Header ceheader = entity.getContentEncoding(); - if (ceheader == null) return; - for (HeaderElement codec : ceheader.getElements()) { - if (codec.getName().equalsIgnoreCase("gzip")) { - response.setEntity(new GzipDecompressingEntity(response.getEntity())); - return; - } - } - }); - } - - @Override - protected HttpRequestExecutor createRequestExecutor() { - return new HttpRequestExecutor(); - } - - @Override - protected HttpRoutePlanner createHttpRoutePlanner() { - return new DefaultHttpRoutePlanner(getConnectionManager().getSchemeRegistry()); - } - - @Override - protected HttpRequestRetryHandler createHttpRequestRetryHandler() { - return new SearcherHttpRequestRetryHandler(retries); - } - } - - /** A retry handler which avoids retrying forever on errors misclassified as transient */ - private static class SearcherHttpRequestRetryHandler implements HttpRequestRetryHandler { - private final int retries; - - public SearcherHttpRequestRetryHandler(int retries) { - this.retries = retries; - } - - @Override - public boolean retryRequest(IOException e, int executionCount, HttpContext httpContext) { - if (e == null) { - throw new IllegalArgumentException("Exception parameter may not be null"); - } - if (executionCount > retries) { - return false; - } - if (e instanceof NoHttpResponseException) { - // Retry if the server dropped connection on us - return true; - } - if (e instanceof InterruptedIOException) { - // Timeout from federation layer - return false; - } - if (e instanceof UnknownHostException) { - // Unknown host - return false; - } - if (e instanceof SSLHandshakeException) { - // SSL handshake exception - return false; - } - return true; - } - - - } - - private static class SearcherHttpRequest extends HttpRequestBase { - String method; - - public SearcherHttpRequest(String method, final URI uri) { - super(); - this.method = method; - setURI(uri); - } - - @Override - public String getMethod() { - return method; - } - } - - /** - * Only for testing. - */ - public void shutdownConnectionManagers() { - ClientConnectionManager manager; - if (sharedConnectionManager != null) { - manager = sharedConnectionManager; - } else { - manager = singleClientConnManagerThreadLocal.get(); - } - if (manager != null) { - manager.shutdown(); - } - } - - protected static final class ThrowingCertificateStore implements CertificateStore { - - @Override - public String getCertificate(String key, long ttl, long retry) { - throw new UnsupportedOperationException("A certificate store is not available"); - } - - } - -} - diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java deleted file mode 100644 index 03ffe2b8a9c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import org.apache.http.Header; -import org.apache.http.HttpEntity; - -/** - * Wrapper for adding timeout to an HttpEntity instance. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedHttpEntity implements HttpEntity { - /** - * The wrapped entity. Never null. - */ - private final HttpEntity entity; - private final long startTime; - private final long timeout; - - public TimedHttpEntity(HttpEntity entity, long startTime, long timeout) { - if (entity == null) { - throw new IllegalArgumentException("TimedHttpEntity cannot be instantiated with null HttpEntity."); - } - this.entity = entity; - this.startTime = startTime; - this.timeout = timeout; - } - - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - InputStream content = entity.getContent(); - if (content == null) { - return null; - } else { - return new TimedStream(content, startTime, timeout); - } - } - - - // START OF PURE FORWARDING METHODS - @Override - public void consumeContent() throws IOException { - entity.consumeContent(); - } - - - @Override - public Header getContentEncoding() { - return entity.getContentEncoding(); - } - - @Override - public long getContentLength() { - return entity.getContentLength(); - } - - @Override - public Header getContentType() { - return entity.getContentType(); - } - - @Override - public boolean isChunked() { - return entity.isChunked(); - } - - @Override - public boolean isRepeatable() { - return entity.isRepeatable(); - } - - @Override - public boolean isStreaming() { - return entity.isStreaming(); - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - entity.writeTo(outstream); - } - // END OF PURE FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java deleted file mode 100644 index 77a42ee0a34..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; - -/** - * A stream which throws a TimeoutException if query timeout has been reached. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedStream extends InputStream { - - /** - * A time barrier value, the point in time from which on read operations will cause an exception. - */ - private final long limit; - - /** - * A wrapped InputStream instance. - */ - private final InputStream content; - - /** - * Wrap an InputStream to make read operations potentially fire off - * TimeoutException. - * - * <p>Typical use would be<br> - * <code>new TimedStream(httpEntity.getContent(), query.getStartTime(), query.getTimeout())</code> - * - * @param content - * the InputStream to wrap - * @param startTime - * start time of query - * @param timeout - * how long the query is allowed to run - */ - public TimedStream(InputStream content, long startTime, long timeout) { - if (content == null) { - throw new IllegalArgumentException("Cannot instantiate TimedStream with null InputStream"); - } - this.content = content; - // The reasion for doing it in here instead of outside the constructor - // is this makes the usage of the class more intuitive IMHO - this.limit = startTime + timeout; - } - - private void checkTime(String message) { - if (System.currentTimeMillis() >= limit) { - throw new TimeoutException(message); - } - } - - // START FORWARDING METHODS: - // All methods below are forwarding methods to the contained stream, where - // some do a timeout check. - @Override - public int read() throws IOException { - int data = content.read(); - checkTime("Timed out during read()."); - return data; - } - - @Override - public int available() throws IOException { - return content.available(); - } - - @Override - public void close() throws IOException { - content.close(); - } - - @Override - public synchronized void mark(int readlimit) { - content.mark(readlimit); - } - - @Override - public boolean markSupported() { - return content.markSupported(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int length = content.read(b, off, len); - checkTime("Timed out during read(byte[], int, int)"); - return length; - } - - @Override - public int read(byte[] b) throws IOException { - int length = content.read(b); - checkTime("Timed out during read(byte[])"); - return length; - } - - @Override - public synchronized void reset() throws IOException { - content.reset(); - } - - @Override - public long skip(long n) throws IOException { - long skipped = content.skip(n); - checkTime("Timed out during skip(long)"); - return skipped; - } - // END FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java deleted file mode 100644 index 18dd59ec37b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Timeout marker for slow HTTP connections. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimeoutException extends RuntimeException { - - /** - * Auto-generated version ID. - */ - private static final long serialVersionUID = 7084147598258586559L; - - public TimeoutException(String message) { - super(message); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java deleted file mode 100644 index af334b35221..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.search.federation.http; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java deleted file mode 100644 index 3a0db9b76de..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import java.util.Iterator; - -import com.yahoo.prelude.query.*; - -/** - * Marshal a query stack into an advanced query string suitable for - * passing to another QRS. - * - * @author Steinar Knutsen - * @author Rong-En Fan - * @deprecated use YQL - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QueryMarshaller { - - private boolean atRoot = true; - - public String marshal(Item root) { - if (root == null || root instanceof NullItem) { - return null; - } - StringBuilder s = new StringBuilder(); - marshal(root, s); - atRoot = true; - return s.toString(); - } - - /** - * We do not yet care about exact match indices - */ - private void marshal(Item root, StringBuilder s) { - switch (root.getItemType()) { - case OR: - marshalOr((OrItem) root, s); - break; - case AND: - marshalAnd((CompositeItem) root, s); - break; - case NOT: - marshalNot((NotItem) root, s); - break; - case RANK: - marshalRank((RankItem) root, s); - break; - case WORD: - case INT: - case PREFIX: - case SUBSTRING: - case SUFFIX: - marshalWord((TermItem) root, s); - break; - case PHRASE: - // PhraseItem and PhraseSegmentItem don't add quotes for segmented - // termse - if (root instanceof PhraseSegmentItem) { - marshalPhrase((PhraseSegmentItem) root, s); - } else { - marshalPhrase((PhraseItem) root, s); - } - break; - case NEAR: - marshalNear((NearItem) root, s); - break; - case ONEAR: - marshalNear((ONearItem) root, s); - break; - case WEAK_AND: - marshalWeakAnd((WeakAndItem)root, s); - break; - default: - break; - } - } - - - private void marshalWord(TermItem item, StringBuilder s) { - String index = item.getIndexName(); - if (index.length() != 0) { - s.append(item.getIndexName()).append(':'); - } - s.append(item.stringValue()); - if (item.getWeight() != Item.DEFAULT_WEIGHT) - s.append("!").append(item.getWeight()); - } - - private void marshalRank(RankItem root, StringBuilder s) { - marshalComposite("RANK", root, s); - } - - private void marshalNot(NotItem root, StringBuilder s) { - marshalComposite("ANDNOT", root, s); - } - - private void marshalOr(OrItem root, StringBuilder s) { - marshalComposite("OR", root, s); - } - - /** - * Dump WORD items, and add space between each of them unless those - * words came from segmentation. - * - * @param root CompositeItem - * @param s current marshaled query - */ - private void dumpWords(CompositeItem root, StringBuilder s) { - for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) { - Item word = i.next(); - boolean useSeparator = true; - if (word instanceof TermItem) { - s.append(((TermItem) word).stringValue()); - if (word instanceof WordItem) { - useSeparator = !((WordItem) word).isFromSegmented(); - } - } else { - dumpWords((CompositeItem) word, s); - } - if (useSeparator && i.hasNext()) { - s.append(' '); - } - } - } - - private void marshalPhrase(PhraseItem root, StringBuilder s) { - marshalPhrase(root, s, root.isExplicit(), false); - } - - private void marshalPhrase(PhraseSegmentItem root, StringBuilder s) { - marshalPhrase(root, s, root.isExplicit(), true); - } - - private void marshalPhrase(IndexedItem root, StringBuilder s, boolean isExplicit, boolean isSegmented) { - String index = root.getIndexName(); - if (index.length() != 0) { - s.append(root.getIndexName()).append(':'); - } - if (isExplicit || !isSegmented) s.append('"'); - dumpWords((CompositeItem) root, s); - if (isExplicit || !isSegmented) s.append('"'); - } - - private void marshalNear(NearItem root, StringBuilder s) { - marshalComposite(root.getName() + "(" + root.getDistance() + ")", root, s); - } - - // Not only AndItem returns ItemType.AND - private void marshalAnd(CompositeItem root, StringBuilder s) { - marshalComposite("AND", root, s); - } - - private void marshalWeakAnd(WeakAndItem root, StringBuilder s) { - marshalComposite("WAND(" + root.getN() + ")", root, s); - } - - private void marshalComposite(String operator, CompositeItem root, StringBuilder s) { - boolean useParen = !atRoot; - if (useParen) { - s.append("( "); - } else { - atRoot = false; - } - for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) { - Item item = i.next(); - marshal(item, s); - if (i.hasNext()) - s.append(' ').append(operator).append(' '); - } - if (useParen) { - s.append(" )"); - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java deleted file mode 100644 index a6aec30b496..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java +++ /dev/null @@ -1,642 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.result.Relevance; -import com.yahoo.text.XML; -import org.xml.sax.*; -import org.xml.sax.helpers.DefaultHandler; -import org.xml.sax.helpers.XMLReaderFactory; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; - -import static com.yahoo.text.Lowercase.toLowerCase; - -/** - * Parse Vespa XML results and create Result instances. - * - * @author Steinar Knutsen - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -@SuppressWarnings("deprecation") -public class ResultBuilder extends DefaultHandler { - private static final String ERROR = "error"; - - private static final String FIELD = "field"; - - private static Logger log = Logger.getLogger(ResultBuilder.class.getName()); - - /** Namespaces feature id (http://xml.org/sax/features/namespaces). */ - protected static final String NAMESPACES_FEATURE_ID = "http://xml.org/sax/features/namespaces"; - - /** - * Namespace prefixes feature id - * (http://xml.org/sax/features/namespace-prefixes). - */ - protected static final String NAMESPACE_PREFIXES_FEATURE_ID = "http://xml.org/sax/features/namespace-prefixes"; - - /** Validation feature id (http://xml.org/sax/features/validation). */ - protected static final String VALIDATION_FEATURE_ID = "http://xml.org/sax/features/validation"; - - /** - * Schema validation feature id - * (http://apache.org/xml/features/validation/schema). - */ - protected static final String SCHEMA_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/schema"; - - /** - * Dynamic validation feature id - * (http://apache.org/xml/features/validation/dynamic). - */ - protected static final String DYNAMIC_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/dynamic"; - - // default settings - - /** Default parser name. */ - protected static final String DEFAULT_PARSER_NAME = "org.apache.xerces.parsers.SAXParser"; - - /** Default namespaces support (false). */ - protected static final boolean DEFAULT_NAMESPACES = false; - - /** Default namespace prefixes (false). */ - protected static final boolean DEFAULT_NAMESPACE_PREFIXES = false; - - /** Default validation support (false). */ - protected static final boolean DEFAULT_VALIDATION = false; - - /** Default Schema validation support (false). */ - protected static final boolean DEFAULT_SCHEMA_VALIDATION = false; - - /** Default dynamic validation support (false). */ - protected static final boolean DEFAULT_DYNAMIC_VALIDATION = false; - - private StringBuilder fieldContent; - - private String fieldName; - - private int fieldLevel = 0; - - private boolean hasLiteralTags = false; - - private Map<String, Object> hitFields = new HashMap<>(); - private String hitType; - private String hitRelevance; - private String hitSource; - - private int offset = 0; - - private List<Tag> tagStack = new ArrayList<>(); - - private final XMLReader parser; - - private Query query; - - private Result result; - - private static enum ResultPart { - ROOT, ERRORDETAILS, HIT, HITGROUP; - } - - Deque<ResultPart> location = new ArrayDeque<>(10); - - private String currentErrorCode; - - private String currentError; - - private Deque<HitGroup> hitGroups = new ArrayDeque<>(5); - - private static class Tag { - public final String name; - - /** - * Offset is a number which is generated for all data and tags inside - * fields, used to determine whether a tag was closed without enclosing - * any characters or other tags. - */ - public final int offset; - - public Tag(final String name, final int offset) { - this.name = name; - this.offset = offset; - } - - @Override - public String toString() { - return name + '(' + Integer.valueOf(offset) + ')'; - } - } - - /** Default constructor. */ - public ResultBuilder() throws RuntimeException { - this(createParser()); - } - - public ResultBuilder(XMLReader parser) { - this.parser = parser; - this.parser.setContentHandler(this); - this.parser.setErrorHandler(this); - } - - public static XMLReader createParser() { - ClassLoader savedContextClassLoader = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(org.apache.xerces.parsers.SAXParser.class.getClassLoader()); - - try { - XMLReader reader = XMLReaderFactory.createXMLReader(DEFAULT_PARSER_NAME); - setParserFeatures(reader); - return reader; - } catch (Exception e) { - throw new RuntimeException("error: Unable to instantiate parser (" - + DEFAULT_PARSER_NAME + ")", e); - } finally { - Thread.currentThread().setContextClassLoader(savedContextClassLoader); - } - } - - private static void setParserFeatures(XMLReader reader) { - try { - reader.setFeature(NAMESPACES_FEATURE_ID, DEFAULT_NAMESPACES); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + NAMESPACES_FEATURE_ID + ")"); - } - try { - reader.setFeature(NAMESPACE_PREFIXES_FEATURE_ID, - DEFAULT_NAMESPACE_PREFIXES); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + NAMESPACE_PREFIXES_FEATURE_ID + ")"); - } - try { - reader.setFeature(VALIDATION_FEATURE_ID, DEFAULT_VALIDATION); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + VALIDATION_FEATURE_ID + ")"); - } - try { - reader.setFeature(SCHEMA_VALIDATION_FEATURE_ID, - DEFAULT_SCHEMA_VALIDATION); - } catch (SAXNotRecognizedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not recognize feature (" - + SCHEMA_VALIDATION_FEATURE_ID + ")"); - - } catch (SAXNotSupportedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + SCHEMA_VALIDATION_FEATURE_ID + ")"); - } - - try { - reader.setFeature(DYNAMIC_VALIDATION_FEATURE_ID, - DEFAULT_DYNAMIC_VALIDATION); - } catch (SAXNotRecognizedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not recognize feature (" - + DYNAMIC_VALIDATION_FEATURE_ID + ")"); - - } catch (SAXNotSupportedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + DYNAMIC_VALIDATION_FEATURE_ID + ")"); - } - } - - @Override - public void startDocument() throws SAXException { - reset(); - result = new Result(query); - hitGroups.addFirst(result.hits()); - location.addFirst(ResultPart.ROOT); - return; - } - - private void reset() { - result = null; - fieldLevel = 0; - hasLiteralTags = false; - tagStack = null; - fieldContent = null; - offset = 0; - currentError = null; - currentErrorCode = null; - hitGroups.clear(); - location.clear(); - } - - @Override - public void startElement(String uri, String local, String raw, - Attributes attrs) throws SAXException { - // "Everybody" wants this switch to be moved into the - // enum class instead, but in this case, I find the classic - // approach more readable. - switch (location.peekFirst()) { - case HIT: - if (fieldLevel > 0) { - tagInField(raw, attrs, FIELD); - ++offset; - return; - } - if (FIELD.equals(raw)) { - ++fieldLevel; - fieldName = attrs.getValue("name"); - fieldContent = new StringBuilder(); - hasLiteralTags = false; - } - break; - case ERRORDETAILS: - if (fieldLevel > 0) { - tagInField(raw, attrs, ERROR); - ++offset; - return; - } - if (ERROR.equals(raw)) { - if (attrs != null) { - currentErrorCode = attrs.getValue("code"); - currentError = attrs.getValue("error"); - } - ++fieldLevel; - fieldContent = new StringBuilder(); - hasLiteralTags = false; - } - break; - case HITGROUP: - if ("hit".equals(raw)) { - startHit(attrs); - } else if ("group".equals(raw)) { - startHitGroup(attrs); - } - break; - case ROOT: - if ("hit".equals(raw)) { - startHit(attrs); - } else if ("errordetails".equals(raw)) { - location.addFirst(ResultPart.ERRORDETAILS); - } else if ("result".equals(raw)) { - if (attrs != null) { - String total = attrs.getValue("total-hit-count"); - if (total != null) { - result.setTotalHitCount(Long.valueOf(total)); - } - } - } else if ("group".equals(raw)) { - startHitGroup(attrs); - } else if (ERROR.equals(raw)) { - if (attrs != null) { - currentErrorCode = attrs.getValue("code"); - fieldContent = new StringBuilder(); - } - } - break; - } - ++offset; - } - - private void startHitGroup(Attributes attrs) { - HitGroup g = new HitGroup(); - Set<String> types = g.types(); - - final String source; - if (attrs != null) { - String groupType = attrs.getValue("type"); - if (groupType != null) { - for (String s : groupType.split(" ")) { - if (s.length() > 0) { - types.add(s); - } - } - } - - source = attrs.getValue("source"); - } else { - source = null; - } - - g.setId((source != null) ? source : "dummy"); - - hitGroups.peekFirst().add(g); - hitGroups.addFirst(g); - location.addFirst(ResultPart.HITGROUP); - } - - private void startHit(Attributes attrs) { - hitFields.clear(); - location.addFirst(ResultPart.HIT); - if (attrs != null) { - hitRelevance = attrs.getValue("relevancy"); - hitSource = attrs.getValue("source"); - hitType = attrs.getValue("type"); - } else { - hitRelevance = null; - hitSource = null; - hitType = null; - } - } - - private void tagInField(String tag, Attributes attrs, String enclosingTag) { - if (!hasLiteralTags) { - hasLiteralTags = true; - String fieldTillNow = XML.xmlEscape(fieldContent.toString(), false); - fieldContent = new StringBuilder(fieldTillNow); - tagStack = new ArrayList<>(); - } - if (enclosingTag.equals(tag)) { - ++fieldLevel; - } - if (tagStack.size() > 0) { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (prevTag != null && (prevTag.offset + 1) == offset) { - fieldContent.append(">"); - } - } - fieldContent.append("<").append(tag); - if (attrs != null) { - int attrCount = attrs.getLength(); - for (int i = 0; i < attrCount; i++) { - fieldContent.append(" ").append(attrs.getQName(i)) - .append("=\"").append( - XML.xmlEscape(attrs.getValue(i), true)).append( - "\""); - } - } - tagStack.add(new Tag(tag, offset)); - } - - private void endElementInField(String qName, String enclosingTag) { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) { - fieldContent.append(" />"); - } else { - fieldContent.append("</").append(qName).append('>'); - } - if (prevTag.name.equals(qName)) { - tagStack.remove(tagStack.size() - 1); - } - } - - private void endElementInHitField(String qName) { - if (FIELD.equals(qName) && --fieldLevel == 0) { - Object content; - if (hasLiteralTags) { - content = new XMLString(fieldContent.toString()); - } else { - content = fieldContent.toString(); - } - hitFields.put(fieldName, content); - if ("collapseId".equals(fieldName)) { - hitFields.put(fieldName, Integer.valueOf(content.toString())); - } - fieldName = null; - fieldContent = null; - tagStack = null; - } else { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) { - fieldContent.append(" />"); - } else { - fieldContent.append("</").append(qName).append('>'); - } - if (prevTag.name.equals(qName)) { - tagStack.remove(tagStack.size() - 1); - } - } - } - @Override - public void characters(char ch[], int start, int length) - throws SAXException { - - switch (location.peekFirst()) { - case ERRORDETAILS: - case HIT: - if (fieldLevel > 0) { - if (hasLiteralTags) { - if (tagStack.size() > 0) { - Tag tag = tagStack.get(tagStack.size() - 1); - if (tag != null && (tag.offset + 1) == offset) { - fieldContent.append(">"); - } - } - fieldContent.append( - XML.xmlEscape(new String(ch, start, length), false)); - } else { - fieldContent.append(ch, start, length); - } - } - break; - default: - if (fieldContent != null) { - fieldContent.append(ch, start, length); - } - break; - } - ++offset; - } - - @Override - public void ignorableWhitespace(char ch[], int start, int length) - throws SAXException { - return; - } - - @Override - public void processingInstruction(String target, String data) - throws SAXException { - return; - } - - @Override - public void endElement(String namespaceURI, String localName, String qName) - throws SAXException { - switch (location.peekFirst()) { - case HITGROUP: - if ("group".equals(qName)) { - hitGroups.removeFirst(); - location.removeFirst(); - } - break; - case HIT: - if (fieldLevel > 0) { - endElementInHitField(qName); - } else if ("hit".equals(qName)) { - //assert(hitKeys.size() == hitValues.size()); - //We try to get either uri or documentID and use that as id - Object docId = extractDocumentID(); - Hit newHit = new Hit(docId.toString()); - if (hitRelevance != null) newHit.setRelevance(new Relevance(Double.parseDouble(hitRelevance))); - if(hitSource != null) newHit.setSource(hitSource); - if(hitType != null) { - for(String type: hitType.split(" ")) { - newHit.types().add(type); - } - } - for(Map.Entry<String, Object> field : hitFields.entrySet()) { - newHit.setField(field.getKey(), field.getValue()); - } - - hitGroups.peekFirst().add(newHit); - location.removeFirst(); - } - break; - case ERRORDETAILS: - if (fieldLevel == 1 && ERROR.equals(qName)) { - ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode), - currentError, - fieldContent.toString()); - hitGroups.peekFirst().addError(error); - currentError = null; - currentErrorCode = null; - fieldContent = null; - tagStack = null; - fieldLevel = 0; - } else if (fieldLevel > 0) { - endElementInField(qName, ERROR); - } else if ("errordetails".equals(qName)) { - location.removeFirst(); - } - break; - case ROOT: - if (ERROR.equals(qName)) { - ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode), - fieldContent.toString()); - hitGroups.peekFirst().addError(error); - currentErrorCode = null; - fieldContent = null; - } - break; - default: - break; - } - ++offset; - } - - private Object extractDocumentID() { - Object docId = null; - if (hitFields.containsKey("uri")) { - docId = hitFields.get("uri"); - } else { - final String documentId = "documentId"; - if (hitFields.containsKey(documentId)) { - docId = hitFields.get(documentId); - } else { - final String lcDocumentId = toLowerCase(documentId); - for (Map.Entry<String, Object> e : hitFields.entrySet()) { - String key = e.getKey(); - // case insensitive matching, checking length first hoping to avoid some lowercasing - if (documentId.length() == key.length() && lcDocumentId.equals(toLowerCase(key))) { - docId = e.getValue(); - break; - } - } - } - } - if (docId == null) { - docId = "dummy"; - log.info("Results from vespa backend did not contain either uri or documentId"); - } - return docId; - } - - @Override - public void warning(SAXParseException ex) throws SAXException { - printError("Warning", ex); - } - - @Override - public void error(SAXParseException ex) throws SAXException { - printError("Error", ex); - } - - @Override - public void fatalError(SAXParseException ex) throws SAXException { - printError("Fatal Error", ex); - // throw ex; - } - - /** Prints the error message. */ - protected void printError(String type, SAXParseException ex) { - StringBuilder errorMessage = new StringBuilder(); - - errorMessage.append(type); - if (ex != null) { - String systemId = ex.getSystemId(); - if (systemId != null) { - int index = systemId.lastIndexOf('/'); - if (index != -1) - systemId = systemId.substring(index + 1); - errorMessage.append(' ').append(systemId); - } - } - errorMessage.append(':') - .append(ex.getLineNumber()) - .append(':') - .append(ex.getColumnNumber()) - .append(": ") - .append(ex.getMessage()); - log.log(LogLevel.WARNING, errorMessage.toString()); - - } - - public Result parse(String identifier, Query query) { - Result toReturn; - - setQuery(query); - try { - parser.parse(identifier); - } catch (SAXParseException e) { - // ignore - } catch (Exception e) { - log.log(LogLevel.WARNING, "Error parsing result from Vespa",e); - Exception se = e; - if (e instanceof SAXException) { - se = ((SAXException) e).getException(); - } - if (se != null) - se.printStackTrace(System.err); - else - e.printStackTrace(System.err); - } - toReturn = result; - reset(); - return toReturn; - } - - public Result parse(InputSource input, Query query) { - Result toReturn; - - setQuery(query); - try { - parser.parse(input); - } catch (SAXParseException e) { - // ignore - } catch (Exception e) { - log.log(LogLevel.WARNING, "Error parsing result from Vespa",e); - Exception se = e; - if (e instanceof SAXException) { - se = ((SAXException) e).getException(); - } - if (se != null) - se.printStackTrace(System.err); - else - e.printStackTrace(System.err); - } - toReturn = result; - reset(); - return toReturn; - } - - - private void setQuery(Query query) { - this.query = query; - } -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java deleted file mode 100644 index 246732d0970..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Map; -import java.util.Set; - -import com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher; -import org.xml.sax.InputSource; -import org.xml.sax.XMLReader; - -import com.google.inject.Inject; -import com.yahoo.collections.Tuple2; -import com.yahoo.component.ComponentId; -import com.yahoo.component.Version; -import com.yahoo.component.chain.dependencies.After; -import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.QueryCanonicalizer; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.federation.http.Connection; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.textserialize.TextSerialize; -import com.yahoo.search.yql.MinimalQueryInserter; -import com.yahoo.statistics.Statistics; - -import edu.umd.cs.findbugs.annotations.Nullable; - -/** - * Backend searcher for external Vespa clusters (queried over http). - * - * <p>If the "sources" argument should be honored on an external cluster - * when using YQL+, override {@link #chooseYqlSources(Set)}.</p> - * - * @author Arne Bergene Fossaa - * @author Steinar Knutsen - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated -@Provides("Vespa") -@After("*") -public class VespaSearcher extends ConfiguredHTTPProviderSearcher { - - private final ThreadLocal<XMLReader> readerHolder = new ThreadLocal<>(); - private final Query.Type queryType; - private final Tuple2<String, Version> segmenterVersion; - - private static final CompoundName select = new CompoundName("select"); - private static final CompoundName streamingUserid = new CompoundName("streaming.userid"); - private static final CompoundName streamingGroupname = new CompoundName("streaming.groupname"); - private static final CompoundName streamingSelection = new CompoundName("streaming.selection"); - - /** Create an instance from configuration */ - public VespaSearcher(ComponentId id, ProviderConfig config, QrBinaryCacheConfig c, - QrBinaryCacheRegionConfig r, Statistics statistics) { - this(id, config, c, r, statistics, null); - } - - /** - * Create an instance from configuration - * - * @param linguistics used for generating meta info for YQL+ - */ - @Inject - public VespaSearcher(ComponentId id, ProviderConfig config, - QrBinaryCacheConfig c, QrBinaryCacheRegionConfig r, - Statistics statistics, @Nullable Linguistics linguistics) { - super(id, config, c, r, statistics); - queryType = toQueryType(config.queryType()); - if (linguistics == null) { - segmenterVersion = null; - } else { - segmenterVersion = linguistics.getVersion(Linguistics.Component.SEGMENTER); - } - } - - /** - * Create an instance from direct parameters having a single connection. - * Useful for testing - */ - public VespaSearcher(String idString, String host, int port, String path) { - super(idString, host, port, path, Statistics.nullImplementation); - queryType = toQueryType(ProviderConfig.QueryType.LEGACY); - segmenterVersion = null; - } - - void addProperty(Map<String, String> queryMap, Query query, CompoundName property) { - Object o = query.properties().get(property); - if (o != null) { - queryMap.put(property.toString(), o.toString()); - } - } - - @Override - public Map<String, String> getQueryMap(Query query) { - Map<String, String> queryMap = getQueryMapWithoutHitsOffset(query); - queryMap.put("offset", Integer.toString(query.getOffset())); - queryMap.put("hits", Integer.toString(query.getHits())); - queryMap.put("presentation.format", "xml"); - - addProperty(queryMap, query, select); - addProperty(queryMap, query, streamingUserid); - addProperty(queryMap, query, streamingGroupname); - addProperty(queryMap, query, streamingSelection); - return queryMap; - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return getQueryMapWithoutHitsOffset(q); - } - - private Map<String, String> getQueryMapWithoutHitsOffset(Query query) { - Map<String, String> queryMap = super.getQueryMap(query); - if (queryType == Query.Type.YQL) { - queryMap.put(MinimalQueryInserter.YQL.toString(), marshalQuery(query)); - } else { - queryMap.put("query", marshalQuery(query.getModel().getQueryTree())); - queryMap.put("type", queryType.toString()); - } - - addNonExcludedSourceProperties(query, queryMap); - return queryMap; - } - - Query.Type toQueryType(ProviderConfig.QueryType.Enum providerQueryType) { - if (providerQueryType == ProviderConfig.QueryType.LEGACY) { - return Query.Type.ADVANCED; - } else if (providerQueryType == ProviderConfig.QueryType.PROGRAMMATIC) { - return Query.Type.PROGRAMMATIC; - } else if (providerQueryType == ProviderConfig.QueryType.YQL) { - return Query.Type.YQL; - } else if (providerQueryType == ProviderConfig.QueryType.SELECT) { - return Query.Type.SELECT; - } else { - throw new RuntimeException("Query type " + providerQueryType + " unsupported."); - } - } - - /** - * Serialize the query parameter for outgoing queries. For YQL+ queries, - * sources and fields will be set to all sources and all fields, to follow - * the behavior of other query types. - * - * @param query - * the current, outgoing query - * @return a string to include in an HTTP request - */ - public String marshalQuery(Query query) { - if (queryType != Query.Type.YQL) { - return marshalQuery(query.getModel().getQueryTree()); - } - - query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once - Query workQuery = query.clone(); - String error = QueryCanonicalizer.canonicalize(workQuery); - if (error != null) { - getLogger().log(LogLevel.WARNING,"Could not normalize [" + query.toString() + "]: " + error); - // Just returning null here is the pattern from existing code... - return null; - } - chooseYqlSources(workQuery.getModel().getSources()); - chooseYqlSummaryFields(workQuery.getPresentation().getSummaryFields()); - return workQuery.yqlRepresentation(getSegmenterVersion(), false); - } - - public String marshalQuery(QueryTree root) { - QueryTree rootClone = root.clone(); // TODO: Why? - String error = QueryCanonicalizer.canonicalize(rootClone); - if (error != null) return null; - - return marshalRoot(rootClone.getRoot()); - } - - private String marshalRoot(Item root) { - switch (queryType) { - case ADVANCED: return new QueryMarshaller().marshal(root); - case PROGRAMMATIC: return TextSerialize.serialize(root); - default: throw new RuntimeException("Unsupported query type."); - } - } - - private XMLReader getReader() { - XMLReader reader = readerHolder.get(); - if (reader == null) { - reader = ResultBuilder.createParser(); - readerHolder.set(reader); - } - return reader; - } - - @Override - public void unmarshal(InputStream stream, long contentLength, Result result) { - ResultBuilder parser = new ResultBuilder(getReader()); - Result mResult = parser.parse(new InputSource(stream), - result.getQuery()); - result.mergeWith(mResult); - result.hits().addAll(mResult.hits().asUnorderedHits()); - } - - /** Returns the canonical Vespa ping URI, http://host:port/status.html */ - @Override - public URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException { - return new URL(getParameters().getSchema(), connection.getHost(), - connection.getPort(), "/status.html").toURI(); - } - - /** - * Get the segmenter version data used when creating YQL queries. Useful if - * overriding {@link #marshalQuery(Query)}. - * - * @return a tuple with the name of the segmenting engine in use, and its - * version - */ - protected Tuple2<String, Version> getSegmenterVersion() { - return segmenterVersion; - } - - /** - * Choose which source arguments to use for the external cluster when - * generating a YQL+ query string. This is called from - * {@link #marshalQuery(Query)}. The default implementation clears the set, - * i.e. requests all sources. Other implementations may modify the source - * set as they see fit, or simply do nothing. - * - * @param sources - * the set of source names to use for the outgoing query - */ - protected void chooseYqlSources(Set<String> sources) { - sources.clear(); - } - - /** - * Choose which summary fields to request from the external cluster when - * generating a YQL+ query string. This is called from - * {@link #marshalQuery(Query)}. The default implementation clears the set, - * i.e. requests all fields. Other implementations may modify the summary - * field set as they see fit, or simply do nothing. - * - * @param summaryFields - * the set of source names to use for the outgoing query - */ - protected void chooseYqlSummaryFields(Set<String> summaryFields) { - summaryFields.clear(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java deleted file mode 100644 index a1c9236ff9c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.search.federation.vespa; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java index 83eca89dffa..13c23234910 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingRequest.java @@ -121,20 +121,6 @@ public class GroupingRequest { } /** - * Sets the result {@link RootGroup} of this request. This is used by the executing grouping searcher, and should - * not be called by a requesting searcher. - * - * @param group the result to set. - * @return this, to allow chaining. - * @deprecated this is a noop - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public GroupingRequest setResultGroup(RootGroup group) { - return this; - } - - /** * Returns the list of {@link Continuation}s of this request. This is used by the executing grouping searcher to * allow pagination of grouping results. * @@ -156,21 +142,6 @@ public class GroupingRequest { return newRequest; } - /** - * Returns all instances of this class that have been attached to the given {@link Query}. If no requests have been - * attached to the {@link Query}, this method returns an empty list. - * - * @param query the query whose requests to return. - * @return the list of grouping requests. - * @deprecated use query.getSelect().getGrouping() - */ - @SuppressWarnings({ "unchecked" }) - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static List<GroupingRequest> getRequests(Query query) { - return query.getSelect().getGrouping(); - } - @Override public String toString() { return root == null ? "(empty)" : root.toString(); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java b/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java deleted file mode 100644 index 8ca772b9188..00000000000 --- a/container-search/src/main/java/com/yahoo/search/grouping/request/YmumValue.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.grouping.request; - -/** - * This class represents a document checksum in a {@link GroupingExpression}. It evaluates to the YMUM checksum of the - * input {@link com.yahoo.search.result.Hit}. - * - * @author Simon Thoresen Hult - * @author bratseth - */ -public class YmumValue extends DocumentValue { - - /** - * Constructs a new instance of this class. - */ - public YmumValue() { - this(null, null); - } - - private YmumValue(String label, Integer level) { - super("ymum()", label, level); - } - - @Override - public YmumValue copy() { - return new YmumValue(getLabel(), getLevelOrNull()); - } - -} - diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java index 07729b4524a..e1f74df7172 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java @@ -30,9 +30,9 @@ public abstract class BucketGroupId<T> extends GroupId { * * @param type The type of this id's value. * @param from The inclusive-from of the range. - * @param fromImage The String representation of the <tt>from</tt> argument. + * @param fromImage The String representation of the <code>from</code> argument. * @param to The exclusive-to of the range. - * @param toImage The String representation of the <tt>to</tt> argument. + * @param toImage The String representation of the <code>to</code> argument. */ public BucketGroupId(String type, T from, String fromImage, T to, String toImage) { super(type, fromImage, toImage); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java index bebd3fd85b0..a76af885180 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/ValueGroupId.java @@ -27,7 +27,7 @@ public abstract class ValueGroupId<T> extends GroupId { * * @param type The type of this id's value. * @param value The identifying value. - * @param valueImage The String representation of the <tt>value</tt> argument. + * @param valueImage The String representation of the <code>value</code> argument. */ public ValueGroupId(String type, T value, String valueImage) { super(type, valueImage); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java index d017fe2edb3..d5d1bb5b18e 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ExpressionConverter.java @@ -85,7 +85,6 @@ import com.yahoo.search.grouping.request.XorAggregator; import com.yahoo.search.grouping.request.XorBitFunction; import com.yahoo.search.grouping.request.XorFunction; import com.yahoo.search.grouping.request.YearFunction; -import com.yahoo.search.grouping.request.YmumValue; import com.yahoo.search.grouping.request.ZCurveXFunction; import com.yahoo.search.grouping.request.ZCurveYFunction; import com.yahoo.searchlib.aggregation.AggregationResult; @@ -115,7 +114,6 @@ import com.yahoo.searchlib.expression.FloatBucketResultNode; import com.yahoo.searchlib.expression.FloatBucketResultNodeVector; import com.yahoo.searchlib.expression.FloatResultNode; import com.yahoo.searchlib.expression.GetDocIdNamespaceSpecificFunctionNode; -import com.yahoo.searchlib.expression.GetYMUMChecksumFunctionNode; import com.yahoo.searchlib.expression.IntegerBucketResultNode; import com.yahoo.searchlib.expression.IntegerBucketResultNodeVector; import com.yahoo.searchlib.expression.IntegerResultNode; @@ -522,9 +520,6 @@ class ExpressionConverter { return new XorBitFunctionNode().setNumBits(((XorBitFunction)exp).getNumBits()) .addArg(toExpressionNode(((XorBitFunction)exp).getArg(0))); } - if (exp instanceof YmumValue) { - return new GetYMUMChecksumFunctionNode(); - } throw new UnsupportedOperationException("Can not convert '" + exp + "' of class " + exp.getClass().getName() + " to an expression."); } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index b28e648be78..7c2e774f68b 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -217,7 +217,7 @@ public class GroupingExecutor extends Searcher { baseRoot = origRoot.clone(); } if (query.isTraceable(3) && query.getGroupingSessionCache()) { - query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId(false) + "'.", 3); + query.trace("Grouping in " + (lastPass + 1) + " passes. SessionId='" + query.getSessionId() + "'.", 3); } for (int pass = 0; pass <= lastPass; ++pass) { boolean firstPass = (pass == 0); diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 6f965944bdf..d2490ec9532 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -13,13 +13,11 @@ import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.ChainsConfig; import com.yahoo.container.core.ContainerHttpConfig; -import com.yahoo.container.core.QrTemplatesConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.VespaHeaders; import com.yahoo.container.logging.AccessLog; -import com.yahoo.container.protect.FreezeDetector; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.language.Linguistics; @@ -33,6 +31,7 @@ import com.yahoo.prelude.query.parser.ParseException; import com.yahoo.prelude.query.parser.SpecialTokenRegistry; import com.yahoo.processing.rendering.Renderer; import com.yahoo.processing.request.CompoundName; +import com.yahoo.search.query.ranking.SoftTimeout; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.vespa.config.SlimeUtils; @@ -164,48 +163,6 @@ public class SearchHandler extends LoggingRequestHandler { Optional.empty() : Optional.of( containerHttpConfig.hostResponseHeaderKey()); } - /** @deprecated use the constructor with ContainerHttpConfig */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public SearchHandler( - final ChainsConfig chainsConfig, - final IndexInfoConfig indexInfo, - final QrSearchersConfig clusters, - final SpecialtokensConfig specialtokens, - final Statistics statistics, - final Linguistics linguistics, - final Metric metric, - final ComponentRegistry<Renderer> renderers, - final Executor executor, - final AccessLog accessLog, - final QueryProfilesConfig queryProfileConfig, - final ComponentRegistry<Searcher> searchers) { - this (chainsConfig, indexInfo, clusters, specialtokens, statistics, linguistics, metric, renderers, executor, - accessLog, queryProfileConfig, searchers, new ContainerHttpConfig(new ContainerHttpConfig.Builder())); - } - - /** @deprecated use the constructor without deprecated parameters */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public SearchHandler( - final ChainsConfig chainsConfig, - final IndexInfoConfig indexInfo, - final QrSearchersConfig clusters, - final SpecialtokensConfig specialTokens, - final QrTemplatesConfig ignored, - final FreezeDetector ignored2, - final Statistics statistics, - final Linguistics linguistics, - final Metric metric, - final ComponentRegistry<Renderer> renderers, - final Executor executor, - final AccessLog accessLog, - final QueryProfilesConfig queryProfileConfig, - final ComponentRegistry<Searcher> searchers) { - this(chainsConfig, indexInfo, clusters, specialTokens, statistics, linguistics, metric, renderers, - executor, accessLog, queryProfileConfig, searchers); - } - @Override protected void destroy() { super.destroy(); @@ -263,8 +220,6 @@ public class SearchHandler extends LoggingRequestHandler { Result result = new Result(query, errorMessage); Renderer renderer = getRendererCopy(ComponentSpecification.fromString(request.getProperty("format"))); - result.getTemplating().setRenderer(renderer); // Pre-Vespa 6 Result.getEncoding() expects this TODO: Remove opn Vespa 7 - return new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer); } @@ -291,8 +246,12 @@ public class SearchHandler extends LoggingRequestHandler { Query query = new Query(request, requestMap, queryProfile); - boolean benchmarkOutput = VespaHeaders.benchmarkOutput(request); - boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarkOutput, request.getJDiscRequest().headers()); + boolean benchmarking = VespaHeaders.benchmarkOutput(request); + boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarking, request.getJDiscRequest().headers()); + + // Don't use soft timeout by default when benchmarking to avoid wrong conclusions by excluding nodes + if (benchmarking && ! request.hasProperty(SoftTimeout.enableProperty.toString())) + query.properties().set(SoftTimeout.enableProperty, false); // Find and execute search chain if we have a valid query String invalidReason = query.validate(); @@ -321,22 +280,14 @@ public class SearchHandler extends LoggingRequestHandler { result = search(pathAndQuery, query, searchChain, searchChainRegistry); } - Renderer renderer; - if (result.getTemplating().usesDefaultTemplate()) { // TODO: Remove on Vespa 7 - renderer = toRendererCopy(query.getPresentation().getRenderer()); - result.getTemplating().setRenderer(renderer); // pre-Vespa 6 Result.getEncoding() expects this to be set. - } - else { // somebody explicitly assigned a old style template // TODO: Remove on Vespa 7 - renderer = perRenderingCopy(result.getTemplating().getRenderer()); - } - // Transform result to response - HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), + Renderer renderer = toRendererCopy(query.getPresentation().getRenderer()); + HttpSearchResponse response = new HttpSearchResponse(getHttpResponseStatus(request, result), result, query, renderer); if (hostResponseHeaderKey.isPresent()) response.headers().add(hostResponseHeaderKey.get(), selfHostname); - if (benchmarkOutput) + if (benchmarking) VespaHeaders.benchmarkOutput(response.headers(), benchmarkCoverage, response.getTiming(), response.getHitCounts(), getErrors(result), response.getCoverage()); @@ -390,9 +341,6 @@ public class SearchHandler extends LoggingRequestHandler { } Result result = execution.search(query); - if (result.getTemplating() == null) // TODO: Remove on Vespa 7 - result.getTemplating().setRenderer(renderer); - ensureQuerySet(result, query); execution.fill(result, result.getQuery().getPresentation().getSummary()); diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java index 92e4bb7e5b8..0741e8effac 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/result/PageTemplatesXmlRenderer.java @@ -13,6 +13,7 @@ import com.yahoo.search.Result; import com.yahoo.search.pagetemplates.model.Renderer; import com.yahoo.search.pagetemplates.model.Source; import com.yahoo.search.query.context.QueryContext; +import com.yahoo.search.rendering.XmlRenderer; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.DefaultErrorHit; import com.yahoo.search.result.ErrorHit; @@ -124,7 +125,7 @@ public class PageTemplatesXmlRenderer extends AsynchronousSectionedRenderer<Resu XMLWriter xmlWriter=XMLWriter.from(writer); xmlWriter.openTag("meta").attribute("type", QueryContext.ID); TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root(); - traceRoot.accept(new com.yahoo.search.rendering.DefaultRenderer.RenderingVisitor(xmlWriter, owner.getStartTime())); + traceRoot.accept(new XmlRenderer.RenderingVisitor(xmlWriter, owner.getStartTime())); xmlWriter.closeTag(); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index fd52618ad85..28b605fa095 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -59,7 +59,7 @@ public class Model implements Cloneable { argumentType.addField(new FieldDescription(QUERY_STRING, "string", "query")); argumentType.addField(new FieldDescription(TYPE, "string", "type")); argumentType.addField(new FieldDescription(FILTER, "string","filter")); - argumentType.addField(new FieldDescription(DEFAULT_INDEX, "string", "default-index def-idx defidx")); + argumentType.addField(new FieldDescription(DEFAULT_INDEX, "string", "default-index")); argumentType.addField(new FieldDescription(LANGUAGE, "string", "language lang")); argumentType.addField(new FieldDescription(ENCODING, "string", "encoding")); argumentType.addField(new FieldDescription(SOURCES, "string", "sources search")); @@ -93,34 +93,6 @@ public class Model implements Cloneable { setParent(query); } - /** - * Creates trace a message of language detection results into this Model - * instance's parent query. Do note this will give bogus results if the - * Execution instance is not set correctly. This is done automatically - * inside {@link Execution#search(Query)}. If tracing the same place as - * creating the query instance, {@link #setExecution(Execution)} has to be - * invoked first with the same Execution instance the query is intended to - * be run by. - * - * @deprecated do not use; language can now be assigned later and for parts of the query tree, making this quite useless - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void traceLanguage() { - if (getParent().getTraceLevel() < 2) return; - if (language != null) { - getParent().trace("Language " + getLanguage() + " specified directly as a parameter", false, 2); - } - else { - Language l = getParsingLanguage(); - // Don't include the query, it will trigger query parsing - getParent().trace("Detected language: " + l, false, 2); - getParent().trace("Language " + l + " determined by " + - (Language.fromEncoding(encoding) != Language.UNKNOWN ? "query encoding" : - "the characters in the terms") + ".", false, 2); - } - } - public Language getParsingLanguage() { return getParsingLanguage(queryString); } @@ -535,28 +507,4 @@ public class Model implements Cloneable { return false; } - /** - * Set the YTrace header value to use when transmitting this model to a - * search backend (of some kind). - * - * @param next string representation of header value - * @deprecated not used, ytrace has been discontinued - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setYTraceHeaderToNext(String next) { } - - /** - * Get the YTrace header value to use when transmitting this model to a - * search backend (of some kind). Returns null if no ytrace data is not - * turned on. - * - * @deprecated not used, ytrace has been discontinued - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public String getYTraceHeaderToNext() { - return null; - } - } diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index ae179a2ba07..6b10fd0847c 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -32,7 +32,6 @@ public class Presentation implements Cloneable { public static final String BOLDING = "bolding"; public static final String TIMING = "timing"; public static final String SUMMARY = "summary"; - public static final String REPORT_COVERAGE = "reportCoverage"; public static final String SUMMARY_FIELDS = "summaryFields"; /** The (short) name of the parameter holding the name of the return format to use */ @@ -45,7 +44,6 @@ public class Presentation implements Cloneable { argumentType.addField(new FieldDescription(BOLDING, "boolean", "bolding")); argumentType.addField(new FieldDescription(TIMING, "boolean", "timing")); argumentType.addField(new FieldDescription(SUMMARY, "string", "summary")); - argumentType.addField(new FieldDescription(REPORT_COVERAGE, "string", "reportcoverage")); argumentType.addField(new FieldDescription(FORMAT, "string", "format template")); argumentType.addField(new FieldDescription(SUMMARY_FIELDS, "string", "summaryFields")); argumentType.freeze(); @@ -96,16 +94,6 @@ public class Presentation implements Cloneable { /** Sets whether matching query terms should be bolded in the result */ public void setBolding(boolean bolding) { this.bolding = bolding; } - /** @deprecated coverage information is always returned */ - @Deprecated // OK - // TODO: Remove on Vespa 7 - public boolean getReportCoverage() { return true; } - - /** @deprecated coverage information is always returned */ - @Deprecated // OK - // TODO: Remove on Vespa 7 - public void setReportCoverage(boolean ignored) { } - /** Get the name of the format desired for result rendering. */ @NonNull public ComponentSpecification getRenderer() { return format; } diff --git a/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java b/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java index 49529936901..f57a5f4ab1e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java +++ b/container-search/src/main/java/com/yahoo/search/query/UniqueRequestId.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query; -import com.yahoo.container.Server; - import java.util.concurrent.atomic.AtomicLong; /** @@ -27,17 +25,6 @@ public class UniqueRequestId { /** * Creates a session id which is unique across the cluster this runtime is a member of each time this is called. * Calling this causes synchronization. - * - * @deprecated use nextId(serverId) instead - */ - @Deprecated - public static UniqueRequestId next() { - return new UniqueRequestId(Server.get().getServerDiscriminator(), System.currentTimeMillis(), sequenceCounter.getAndIncrement()); - } - - /** - * Creates a session id which is unique across the cluster this runtime is a member of each time this is called. - * Calling this causes synchronization. */ public static UniqueRequestId next(String serverId) { return new UniqueRequestId(serverId, System.currentTimeMillis(), sequenceCounter.getAndIncrement()); diff --git a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java index e2a025c8fae..6b28b166150 100644 --- a/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java +++ b/container-search/src/main/java/com/yahoo/search/query/context/QueryContext.java @@ -3,7 +3,7 @@ package com.yahoo.search.query.context; import com.yahoo.processing.execution.Execution; import com.yahoo.search.Query; -import com.yahoo.search.rendering.DefaultRenderer; +import com.yahoo.search.rendering.XmlRenderer; import com.yahoo.text.XMLWriter; import com.yahoo.yolean.trace.TraceNode; @@ -88,7 +88,7 @@ public class QueryContext implements Cloneable { XMLWriter xmlWriter=XMLWriter.from(writer); xmlWriter.openTag("meta").attribute("type",ID); TraceNode traceRoot=owner.getModel().getExecution().trace().traceNode().root(); - traceRoot.accept(new DefaultRenderer.RenderingVisitor(xmlWriter,owner.getStartTime())); + traceRoot.accept(new XmlRenderer.RenderingVisitor(xmlWriter, owner.getStartTime())); xmlWriter.closeTag(); } return true; diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java index 7aefa6f0cf2..ca437fb9def 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserEnvironment.java @@ -18,9 +18,7 @@ import com.yahoo.search.searchchain.Execution; public final class ParserEnvironment { private IndexFacts indexFacts = new IndexFacts(); - - @SuppressWarnings("deprecation") - private Linguistics linguistics = new SimpleLinguistics(false); + private Linguistics linguistics = new SimpleLinguistics(); private SpecialTokens specialTokens = new SpecialTokens(); public IndexFacts getIndexFacts() { diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java index 69d46527255..a57c0f98b45 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java @@ -18,7 +18,7 @@ public final class ParserFactory { } /** - * Creates a {@link Parser} appropriate for the given <tt>Query.Type</tt>, providing the Parser with access to + * Creates a {@link Parser} appropriate for the given <code>Query.Type</code>, providing the Parser with access to * the {@link ParserEnvironment} given. * * @param type the query type for which to create a Parser diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index 60427aeb0af..9e043bc3dc9 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -29,12 +29,6 @@ import java.util.Map; */ public class QueryProperties extends Properties { - /** @deprecated use Query.nativeProperties */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static final CompoundName[] PER_SOURCE_QUERY_PROPERTIES = - Query.nativeProperties.toArray(new CompoundName[] {}); - private Query query; private final CompiledQueryProfileRegistry profileRegistry; @@ -119,7 +113,6 @@ public class QueryProperties extends Properties { else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) { if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); - if (key.last().equals(Presentation.REPORT_COVERAGE)) return true; // TODO: Remove this line on Vespa 7 if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat(); if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming(); if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields(); @@ -217,9 +210,9 @@ public class QueryProperties extends Properties { } else if (key.size() == 3 && key.get(1).equals(Ranking.SOFTTIMEOUT)) { SoftTimeout soft = ranking.getSoftTimeout(); - if (key.last().equals(SoftTimeout.ENABLE)) soft.setEnable(asBoolean(value, false)); - if (key.last().equals(SoftTimeout.FACTOR)) soft.setFactor(asDouble(value, 0.50)); - if (key.last().equals(SoftTimeout.TAILCOST)) soft.setTailcost(asDouble(value, 0.10)); + if (key.last().equals(SoftTimeout.ENABLE)) soft.setEnable(asBoolean(value, true)); + if (key.last().equals(SoftTimeout.FACTOR)) soft.setFactor(asDouble(value, null)); + if (key.last().equals(SoftTimeout.TAILCOST)) soft.setTailcost(asDouble(value, null)); } else if (key.size() == 3 && key.get(1).equals(Ranking.MATCHING)) { Matching matching = ranking.getMatching(); @@ -249,7 +242,7 @@ public class QueryProperties extends Properties { query.getPresentation().setTiming(asBoolean(value, true)); else if (key.last().equals(Presentation.SUMMARY_FIELDS)) query.getPresentation().setSummaryFields(asString(value,"")); - else if ( ! key.last().equals(Presentation.REPORT_COVERAGE)) // TODO: Change this line to "else" on Vespa 7 + else throwIllegalParameter(key.last(), Presentation.PRESENTATION); } else if (key.size()==2 && key.first().equals(Select.SELECT)) { diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java index ca6fd44af50..a05ecee8a1e 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.ranking; +import com.yahoo.processing.request.CompoundName; import com.yahoo.search.query.Ranking; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; @@ -21,6 +22,10 @@ public class SoftTimeout implements Cloneable { public static final String FACTOR = "factor"; public static final String TAILCOST = "tailcost"; + /** The full property name for turning softtimeout on or off */ + public static final CompoundName enableProperty = + CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, ENABLE); + static { argumentType = new QueryProfileType(Ranking.SOFTTIMEOUT); argumentType.setStrict(true); @@ -31,14 +36,15 @@ public class SoftTimeout implements Cloneable { } public static QueryProfileType getArgumentType() { return argumentType; } - public Boolean enable = null; + private boolean enabled = true; private Double factor = null; private Double tailcost = null; - public void setEnable(boolean enable) { this.enable = enable; } + public void setEnable(boolean enable) { this.enabled = enable; } - public Boolean getEnable() { return enable; } + public Boolean getEnable() { return enabled; } + /** Override the adaptive factor determined on the content nodes */ public void setFactor(double factor) { if ((factor < 0.0) || (factor > 1.0)) { throw new IllegalArgumentException("factor must be in the range [0.0, 1.0], got " + factor); @@ -48,6 +54,7 @@ public class SoftTimeout implements Cloneable { public Double getFactor() { return factor; } + /** Override the tail cost factor determined on the content nodes */ public void setTailcost(double tailcost) { if ((tailcost < 0.0) || (tailcost > 1.0)) { throw new IllegalArgumentException("tailcost must be in the range [0.0, 1.0], got " + tailcost); @@ -59,15 +66,13 @@ public class SoftTimeout implements Cloneable { /** Internal operation - DO NOT USE */ public void prepare(RankProperties rankProperties) { - if (enable != null) { - rankProperties.put("vespa.softtimeout.enable", String.valueOf(enable)); - } - if (factor != null) { + if ( !enabled) return; + + rankProperties.put("vespa.softtimeout.enable", "true"); + if (factor != null) rankProperties.put("vespa.softtimeout.factor", String.valueOf(factor)); - } - if (tailcost != null) { + if (tailcost != null) rankProperties.put("vespa.softtimeout.tailcost", String.valueOf(tailcost)); - } } @Override @@ -83,7 +88,7 @@ public class SoftTimeout implements Cloneable { @Override public int hashCode() { int hash = 0; - if (enable != null) hash += 11 * enable.hashCode(); + if (enabled) hash += 11; if (factor != null) hash += 13 * factor.hashCode(); if (tailcost != null) hash += 17 * tailcost.hashCode(); return hash; @@ -95,7 +100,7 @@ public class SoftTimeout implements Cloneable { if ( ! (o instanceof SoftTimeout)) return false; SoftTimeout other = (SoftTimeout)o; - if ( ! Objects.equals(this.enable, other.enable)) return false; + if ( ! Objects.equals(this.enabled, other.enabled)) return false; if ( ! Objects.equals(this.factor, other.factor)) return false; if ( ! Objects.equals(this.tailcost, other.tailcost)) return false; return true; diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java b/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java deleted file mode 100644 index d6a88a8f4e7..00000000000 --- a/container-search/src/main/java/com/yahoo/search/querytransform/LegacyCombinator.java +++ /dev/null @@ -1,365 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.language.Language; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.IndexedItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.NotItem; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.RankItem; -import com.yahoo.prelude.query.parser.CustomParser; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.query.Properties; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; - -/** - * Compatibility layer to implement the old multi part query syntax, along with - * the features of QueryCombinator. Do <b>not</b> use both QueryCombinator and - * LegacyCombinator in a single search. - * - * <p> - * A searcher which grabs query parameters of the form - * "defidx.(identifier)=(index name)" and "query.(identifier)=(user query)", - * parses them and adds them as AND items to the query root. - * - * <p> - * If the given default index does not exist in the search definition, the query - * part will be parsed with the settings of the default index set to "". - * - * <p> - * If any of the following arguments exist, they will be used: - * - * <p> - * query.(identifier)=query string<br> - * query.(identifier).operator={"req", "rank", "not"}, where "req" is default<br> - * query.(identifier).defidx=default index<br> - * query.(identifier).type={"all", "any", "phrase", "adv", "web"} where "all" is - * default - * - * <p> - * If both defidx.(identifier) and any of - * query.(identifier).{operator,defidx,type} is present in the query, an - * InvalidQueryParameter error will be added, and the query will be passed - * through untransformed. - * - * @author Steinar Knutsen - */ -@Before({"transformedQuery", "com.yahoo.prelude.querytransform.StemmingSearcher"}) -public class LegacyCombinator extends Searcher { - - private static final String TYPESUFFIX = ".type"; - private static final String OPERATORSUFFIX = ".operator"; - private static final String DEFIDXSUFFIX = ".defidx"; - private static final String DEFIDXPREFIX = "defidx."; - private static final String QUERYPREFIX = "query."; - - private enum Combinator { - REQUIRED("req"), PREFERRED("rank"), EXCLUDED("not"); - - String parameterValue; - - private Combinator(String parameterValue) { - this.parameterValue = parameterValue; - } - - static Combinator getCombinator(String name) { - for (Combinator c : Combinator.values()) { - if (c.parameterValue.equals(name)) { - return c; - } - } - return REQUIRED; - } - } - - private static class QueryPart { - final String query; - final String defaultIndex; - final Combinator operator; - final String identifier; - final Query.Type syntax; - - QueryPart(String identifier, String defaultIndex, String oldIndex, - String operator, String query, String syntax) { - validateArguments(identifier, defaultIndex, oldIndex, - operator,syntax); - this.query = query; - if (defaultIndex != null) { - this.defaultIndex = defaultIndex; - } else { - this.defaultIndex = oldIndex; - } - this.operator = Combinator.getCombinator(operator); - this.identifier = identifier; - this.syntax = Query.Type.getType(syntax); - } - - private static void validateArguments(String identifier, String defaultIndex, - String oldIndex, String operator, String syntax) { - if (defaultIndex == null) { - return; - } - if (oldIndex != null) { - throw new IllegalArgumentException(createErrorMessage(identifier, DEFIDXSUFFIX)); - } - if (operator != null) { - throw new IllegalArgumentException(createErrorMessage(identifier, OPERATORSUFFIX)); - } - if (syntax != null) { - throw new IllegalArgumentException(createErrorMessage(identifier, TYPESUFFIX)); - } - } - - private static String createErrorMessage(String identifier, String legacyArgument) { - return "Cannot set both " + DEFIDXPREFIX + identifier + " and " - + QUERYPREFIX + identifier + legacyArgument + "."; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result - + ((identifier == null) ? 0 : identifier.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - QueryPart other = (QueryPart) obj; - if (identifier == null) { - if (other.identifier != null) - return false; - } else if (!identifier.equals(other.identifier)) - return false; - return true; - } - - @Override - public String toString() { - return "QueryPart(" + identifier + ", " + defaultIndex + ", " - + operator + ", " + syntax + ")"; - } - } - - @Override - public Result search(Query query, Execution execution) { - Set<QueryPart> pieces; - Set<String> usedSources; - IndexFacts indexFacts = execution.context().getIndexFacts(); - try { - pieces = findQuerySnippets(query.properties()); - } catch (IllegalArgumentException e) { - query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator got invalid parameters: " - + e.getMessage())); - return execution.search(query); - } - if (pieces.size() == 0) { - return execution.search(query); - } - IndexFacts.Session session = indexFacts.newSession(query); - Language language = query.getModel().getParsingLanguage(); - addAndItems(language, query, pieces, session, execution.context()); - addRankItems(language, query, pieces, session, execution.context()); - try { - addNotItems(language, query, pieces, session, execution.context()); - } catch (IllegalArgumentException e) { - query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator found only excluding terms, no including.")); - return execution.search(query); - } - query.trace("Adding extra query parts.", true, 2); - return execution.search(query); - } - - private void addNotItems(Language language, Query query, Set<QueryPart> pieces, - IndexFacts.Session session, Execution.Context context) { - for (QueryPart part : pieces) { - if (part.operator != Combinator.EXCLUDED) continue; - - String defaultIndex = defaultIndex(session, part); - Item item = parse(language, query, part, defaultIndex, context); - if (item == null) continue; - - setDefaultIndex(part, defaultIndex, item); - addNotItem(query.getModel().getQueryTree(), item); - } - - } - - private void addNotItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - // errr... no positive branch at all? - throw new IllegalArgumentException("No positive terms for query."); - } else if (root.getClass() == NotItem.class) { - ((NotItem) root).addNegativeItem(item); - } else { - NotItem newRoot = new NotItem(); - newRoot.addPositiveItem(root); - newRoot.addNegativeItem(item); - queryTree.setRoot(newRoot); - } - } - - private void addRankItems(Language language, Query query, Set<QueryPart> pieces, IndexFacts.Session session, Execution.Context context) { - for (QueryPart part : pieces) { - if (part.operator != Combinator.PREFERRED) continue; - - String defaultIndex = defaultIndex(session, part); - Item item = parse(language, query, part, defaultIndex, context); - if (item == null) continue; - - setDefaultIndex(part, defaultIndex, item); - addRankItem(query.getModel().getQueryTree(), item); - } - } - - private void addRankItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - queryTree.setRoot(item); - } else if (root.getClass() == RankItem.class) { - // if no clear recall terms, just set the rank term as recall - ((RankItem) root).addItem(item); - } else { - RankItem newRoot = new RankItem(); - newRoot.addItem(root); - newRoot.addItem(item); - queryTree.setRoot(newRoot); - } - } - - private void addAndItems(Language language, Query query, Iterable<QueryPart> pieces, IndexFacts.Session session, Execution.Context context) { - for (QueryPart part : pieces) { - if (part.operator != Combinator.REQUIRED) continue; - - String defaultIndex = defaultIndex(session, part); - Item item = parse(language, query, part, defaultIndex, context); - if (item == null) continue; - - setDefaultIndex(part, defaultIndex, item); - addAndItem(query.getModel().getQueryTree(), item); - } - } - - private void setDefaultIndex(QueryPart part, String defaultIndex, Item item) { - if (defaultIndex == null) { - assignDefaultIndex(item, part.defaultIndex); - } - } - - private Item parse(Language language, Query query, QueryPart part, String defaultIndex, Execution.Context context) { - Item item = null; - try { - CustomParser parser = (CustomParser)ParserFactory.newInstance( - part.syntax, ParserEnvironment.fromExecutionContext(context)); - item = parser.parse(part.query, null, language, query.getModel().getSources(), - context.getIndexFacts(), defaultIndex); - } catch (RuntimeException e) { - String err = Exceptions.toMessageString(e); - query.trace("Query parser threw an exception: " + err, true, 1); - getLogger().log(LogLevel.WARNING, - "Query parser threw exception in searcher LegacyCombinator for " - + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err); - } - return item; - } - - private String defaultIndex(IndexFacts.Session indexFacts, QueryPart part) { - String defaultIndex; - if (indexFacts.getIndex(part.defaultIndex) == Index.nullIndex) { - defaultIndex = null; - } else { - defaultIndex = part.defaultIndex; - } - return defaultIndex; - } - - private static void addAndItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - queryTree.setRoot(item); - } else if (root.getClass() == AndItem.class) { - ((AndItem) root).addItem(item); - } else { - AndItem newRoot = new AndItem(); - newRoot.addItem(root); - newRoot.addItem(item); - queryTree.setRoot(newRoot); - } - } - - private static void assignDefaultIndex(Item item, String defaultIndex) { - if (item instanceof IndexedItem) { - IndexedItem indexName = (IndexedItem) item; - - if ("".equals(indexName.getIndexName())) { - indexName.setIndexName(defaultIndex); - } - } else if (item instanceof CompositeItem) { - Iterator<Item> items = ((CompositeItem) item).getItemIterator(); - while (items.hasNext()) { - Item i = items.next(); - assignDefaultIndex(i, defaultIndex); - } - } - - } - - private static Set<QueryPart> findQuerySnippets(Properties properties) { - Set<QueryPart> pieces = new HashSet<>(); - for (Map.Entry<String, Object> k : properties.listProperties() - .entrySet()) { - String key = k.getKey(); - if (!key.startsWith(QUERYPREFIX)) { - continue; - } - String name = key.substring(QUERYPREFIX.length()); - if (hasDots(name)) { - continue; - } - String index = properties.getString(DEFIDXPREFIX + name); - String oldIndex = properties.getString(QUERYPREFIX + name - + DEFIDXSUFFIX); - String operator = properties.getString(QUERYPREFIX + name - + OPERATORSUFFIX); - String type = properties.getString(QUERYPREFIX + name + TYPESUFFIX); - pieces.add(new QueryPart(name, index, oldIndex, operator, k - .getValue().toString(), type)); - } - return pieces; - } - - private static boolean hasDots(String name) { - int index = name.indexOf('.', 0); - return index != -1; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java b/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java deleted file mode 100644 index 67cb291a760..00000000000 --- a/container-search/src/main/java/com/yahoo/search/querytransform/QueryCombinator.java +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.querytransform; - -import com.yahoo.component.ComponentId; -import com.yahoo.language.Language; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Index; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.IndexedItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.parser.CustomParser; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.query.Properties; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.yolean.Exceptions; - -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; - -/** - * <p>A searcher which grabs query parameters of the form "defidx.(identifier)=(index name)" and - * "query.(identifier)=(user query)", * parses them and adds them as AND items to the query root.</p> - * - * <p>If the given default index does not exist in the search definition, the query part will be parsed with the - * settings of the default index set to the "".</p> - * - * @author Steinar Knutsen - * @deprecated use YQL - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QueryCombinator extends Searcher { - - private static final String QUERYPREFIX = "query."; - - private static class QueryPart { - final String query; - final String defaultIndex; - - QueryPart(String query, String defaultIndex) { - this.query = query; - this.defaultIndex = defaultIndex; - } - } - - public QueryCombinator(ComponentId id) { - super(id); - } - - @Override - public Result search(Query query, Execution execution) { - Set<QueryPart> pieces = findQuerySnippets(query.properties()); - if (pieces.size() == 0) { - return execution.search(query); - } - addAndItems(query, pieces, execution.context()); - query.trace("Adding extra query parts.", true, 2); - return execution.search(query); - } - - private void addAndItems(Query query, Iterable<QueryPart> pieces, Execution.Context context) { - IndexFacts indexFacts = context.getIndexFacts(); - IndexFacts.Session session = indexFacts.newSession(query); - Set<String> usedSources = new HashSet<>(session.documentTypes()); - Language language = query.getModel().getParsingLanguage(); - for (QueryPart part : pieces) { - String defaultIndex; - Item item = null; - Index index = session.getIndex(part.defaultIndex); - if (index == Index.nullIndex) { - defaultIndex = null; - } else { - defaultIndex = part.defaultIndex; - } - try { - CustomParser parser = (CustomParser)ParserFactory.newInstance(query.getModel().getType(), - ParserEnvironment.fromExecutionContext(context)); - item = parser.parse(part.query, null, language, usedSources, indexFacts, defaultIndex); - } catch (RuntimeException e) { - String err = Exceptions.toMessageString(e); - query.trace("Query parser threw an exception: " + err, true, 1); - getLogger().log(LogLevel.WARNING, - "Query parser threw exception searcher QueryCombinator for " - + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err); - } - if (item == null) { - continue; - } - if (defaultIndex == null) { - assignDefaultIndex(item, part.defaultIndex); - } - addAndItem(query.getModel().getQueryTree(), item); - } - } - - private static void addAndItem(QueryTree queryTree, Item item) { - Item root = queryTree.getRoot(); - // JavaDoc claims I can get null, code gives NullItem... well, well, well... - if (root instanceof NullItem || root == null) { - queryTree.setRoot(item); - } else if (root.getClass() == AndItem.class) { - ((AndItem) root).addItem(item); - } else { - AndItem newRoot = new AndItem(); - newRoot.addItem(root); - newRoot.addItem(item); - queryTree.setRoot(newRoot); - } - } - - private static void assignDefaultIndex(Item item, String defaultIndex) { - if (item instanceof IndexedItem) { - IndexedItem indexName = (IndexedItem) item; - - if ("".equals(indexName.getIndexName())) { - indexName.setIndexName(defaultIndex); - } - } else if (item instanceof CompositeItem) { - Iterator<Item> items = ((CompositeItem) item).getItemIterator(); - while (items.hasNext()) { - Item i = items.next(); - assignDefaultIndex(i, defaultIndex); - } - } - } - - private static Set<QueryPart> findQuerySnippets(Properties properties) { - Set<QueryPart> pieces = new HashSet<>(); - for (Map.Entry<String, Object> k : properties.listProperties().entrySet()) { - String key = k.getKey(); - if (!key.startsWith(QUERYPREFIX)) { - continue; - } - String name = key.substring(QUERYPREFIX.length()); - if (hasDots(name)) { - continue; - } - String index = properties.getString("defidx." + name); - pieces.add(new QueryPart(k.getValue().toString(), index)); - } - return pieces; - } - - private static boolean hasDots(String name) { - int index = name.indexOf('.', 0); - return index != -1; - } -} diff --git a/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java deleted file mode 100644 index 30695338741..00000000000 --- a/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java +++ /dev/null @@ -1,427 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.rendering; - -import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.data.XmlProducer; -import com.yahoo.io.ByteWriter; -import com.yahoo.net.URI; -import com.yahoo.prelude.fastsearch.GroupingListHit; -import com.yahoo.prelude.hitfield.HitField; -import com.yahoo.prelude.hitfield.JSONString; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.processing.rendering.AsynchronousSectionedRenderer; -import com.yahoo.processing.response.Data; -import com.yahoo.processing.response.DataList; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.grouping.result.HitRenderer; -import com.yahoo.search.query.context.QueryContext; -import com.yahoo.search.result.*; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XML; -import com.yahoo.text.XMLWriter; -import com.yahoo.yolean.trace.TraceNode; -import com.yahoo.yolean.trace.TraceVisitor; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.io.Writer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetEncoder; -import java.util.Iterator; -import java.util.concurrent.Executor; -import java.util.stream.Collectors; - -/** - * XML rendering of search results. This is NOT the default (but it once was). - * - * @author Tony Vaagenes - * @deprecated use JsonRenderer instead - */ -@SuppressWarnings({ "rawtypes", "deprecation" }) -@Deprecated // OK -// TODO: Rename to XmlRenderer on Vespa 7 -public final class DefaultRenderer extends AsynchronousSectionedRenderer<Result> { - - public static final String DEFAULT_MIMETYPE = "text/xml"; - public static final String DEFAULT_ENCODING = "utf-8"; - - private static final Utf8String RESULT = new Utf8String("result"); - private static final Utf8String GROUP = new Utf8String("group"); - private static final Utf8String ID = new Utf8String("id"); - private static final Utf8String FIELD = new Utf8String("field"); - private static final Utf8String HIT = new Utf8String("hit"); - private static final Utf8String ERROR = new Utf8String("error"); - private static final Utf8String TOTAL_HIT_COUNT = new Utf8String("total-hit-count"); - private static final Utf8String QUERY_TIME = new Utf8String("querytime"); - private static final Utf8String SUMMARY_FETCH_TIME = new Utf8String("summaryfetchtime"); - private static final Utf8String SEARCH_TIME = new Utf8String("searchtime"); - private static final Utf8String NAME = new Utf8String("name"); - private static final Utf8String CODE = new Utf8String("code"); - private static final Utf8String COVERAGE_DOCS = new Utf8String("coverage-docs"); - private static final Utf8String COVERAGE_NODES = new Utf8String("coverage-nodes"); - private static final Utf8String COVERAGE_FULL = new Utf8String("coverage-full"); - private static final Utf8String COVERAGE = new Utf8String("coverage"); - private static final Utf8String RESULTS_FULL = new Utf8String("results-full"); - private static final Utf8String RESULTS = new Utf8String("results"); - private static final Utf8String TYPE = new Utf8String("type"); - private static final Utf8String RELEVANCY = new Utf8String("relevancy"); - private static final Utf8String SOURCE = new Utf8String("source"); - - - // this is shared between umpteen threads by design - private final CopyOnWriteHashMap<String, Utf8String> fieldNameMap = new CopyOnWriteHashMap<>(); - - private XMLWriter writer; - - public DefaultRenderer() { - this(null); - } - - /** - * Creates an XML renderer using a custom executor. - * Using a custom executor is useful for tests to avoid creating new threads for each renderer registry. - */ - public DefaultRenderer(Executor executor) { - super(executor); - } - - @Override - public void init() { - super.init(); - writer = null; - } - - @Override - public String getEncoding() { - if (getResult() == null - || getResult().getQuery() == null - || getResult().getQuery().getModel().getEncoding() == null) { - return DEFAULT_ENCODING; - } else { - return getResult().getQuery().getModel().getEncoding(); - } - } - - @Override - public String getMimeType() { - return DEFAULT_MIMETYPE; - } - - private XMLWriter wrapWriter(Writer writer) { - return XMLWriter.from(writer, 10, -1); - } - - private void header(XMLWriter writer, Result result) throws IOException { - // TODO: move setting this to Result - writer.xmlHeader(getRequestedEncoding(result.getQuery())); - writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT, String.valueOf(result.getTotalHitCount())); - renderCoverageAttributes(result.getCoverage(false), writer); - renderTime(writer, result); - writer.closeStartTag(); - } - - private void renderTime(XMLWriter writer, Result result) { - if ( ! result.getQuery().getPresentation().getTiming()) return; - - final String threeDecimals = "%.3f"; - final double milli = .001d; - final long now = System.currentTimeMillis(); - final long searchTime = now - result.getElapsedTime().first(); - final double searchSeconds = ((double) searchTime) * milli; - - if (result.getElapsedTime().firstFill() != 0L) { - final long queryTime = result.getElapsedTime().weightedSearchTime(); - final long summaryFetchTime = result.getElapsedTime().weightedFillTime(); - final double querySeconds = ((double) queryTime) * milli; - final double summarySeconds = ((double) summaryFetchTime) * milli; - writer.attribute(QUERY_TIME, String.format(threeDecimals, querySeconds)); - writer.attribute(SUMMARY_FETCH_TIME, String.format(threeDecimals, summarySeconds)); - } - writer.attribute(SEARCH_TIME, String.format(threeDecimals, searchSeconds)); - } - - protected static void renderCoverageAttributes(Coverage coverage, XMLWriter writer) throws IOException { - if (coverage == null) return; - writer.attribute(COVERAGE_DOCS,coverage.getDocs()); - writer.attribute(COVERAGE_NODES,coverage.getNodes()); - writer.attribute(COVERAGE_FULL,coverage.getFull()); - writer.attribute(COVERAGE,coverage.getResultPercentage()); - writer.attribute(RESULTS_FULL,coverage.getFullResultSets()); - writer.attribute(RESULTS,coverage.getResultSets()); - } - - public void error(XMLWriter writer, Result result) throws IOException { - ErrorMessage error = result.hits().getError(); - writer.openTag(ERROR).attribute(CODE,error.getCode()).content(error.getMessage(),false).closeTag(); - } - - @SuppressWarnings("UnusedParameters") - protected void emptyResult(XMLWriter writer, Result result) throws IOException {} - - @SuppressWarnings("UnusedParameters") - public void queryContext(XMLWriter writer, QueryContext queryContext, Query owner) throws IOException { - if (owner.getTraceLevel()!=0) { - XMLWriter xmlWriter=XMLWriter.from(writer); - xmlWriter.openTag("meta").attribute("type", QueryContext.ID); - TraceNode traceRoot = owner.getModel().getExecution().trace().traceNode().root(); - traceRoot.accept(new RenderingVisitor(xmlWriter, owner.getStartTime())); - xmlWriter.closeTag(); - } - } - - private void renderSingularHit(XMLWriter writer, Hit hit) { - writer.openTag(HIT); - renderHitAttributes(writer, hit); - writer.closeStartTag(); - renderHitFields(writer, hit); - } - - private void renderHitFields(XMLWriter writer, Hit hit) { - renderSyntheticRelevanceField(writer, hit); - hit.forEachField((name, value) -> renderField(writer, name, value)); - } - - private void renderField(XMLWriter writer, String name, Object value) { - if (name.startsWith("$")) return; - - writeOpenFieldElement(writer, name); - renderFieldContent(writer, value); - writeCloseFieldElement(writer); - } - - private void renderFieldContent(XMLWriter writer, Object value) { - writer.escapedContent(asXML(value), false); - } - - private String asXML(Object value) { - if (value == null) - return "(null)"; - else if (value instanceof XmlProducer) - return ((XmlProducer)value).toXML(); - else if (value instanceof HitField) - return ((HitField)value).quotedContent(false); - else if (value instanceof StructuredData || value instanceof XMLString || value instanceof JSONString) - return value.toString(); - else - return XML.xmlEscape(value.toString(), false, '\u001f'); - } - - private void renderSyntheticRelevanceField(XMLWriter writer, Hit hit) { - String relevancyFieldName = "relevancy"; - Relevance relevance = hit.getRelevance(); - - // skip depending on hit type - if (relevance != null) { - renderSimpleField(writer, relevancyFieldName, relevance); - } - } - - private void renderSimpleField(XMLWriter writer, String relevancyFieldName, Relevance relevance) { - writeOpenFieldElement(writer, relevancyFieldName); - writer.content(relevance.toString(), false); - writeCloseFieldElement(writer); - } - - private void writeCloseFieldElement(XMLWriter writer) { - writer.closeTag(); - } - - private void writeOpenFieldElement(XMLWriter writer, String relevancyFieldName) { - Utf8String utf8 = fieldNameMap.get(relevancyFieldName); - if (utf8 == null) { - utf8 = new Utf8String(relevancyFieldName); - fieldNameMap.put(relevancyFieldName, utf8); - } - writer.openTag(FIELD).attribute(NAME, utf8); - writer.closeStartTag(); - } - - private void renderHitAttributes(XMLWriter writer, Hit hit) { - writer.attribute(TYPE, hit.types().stream().collect(Collectors.joining(" "))); - if (hit.getRelevance() != null) - writer.attribute(RELEVANCY, hit.getRelevance().toString()); - writer.attribute(SOURCE, hit.getSource()); - } - - private void renderHitGroup(XMLWriter writer, HitGroup hit) throws IOException { - if (HitRenderer.renderHeader(hit, writer)) { - // empty - } else if (hit.types().contains("grouphit")) { - // TODO Keep this? - renderHitGroupOfTypeGroupHit(writer, hit); - } else { - renderGroup(writer, hit); - } - } - - private void renderGroup(XMLWriter writer, HitGroup hit) { - writer.openTag(GROUP); - renderHitAttributes(writer, hit); - writer.closeStartTag(); - } - - private void renderHitGroupOfTypeGroupHit(XMLWriter writer, HitGroup hit) { - writer.openTag(HIT); - renderHitAttributes(writer, hit); - renderId(writer, hit); - writer.closeStartTag(); - } - - private void renderId(XMLWriter writer, HitGroup hit) { - URI uri = hit.getId(); - if (uri != null) { - writer.openTag(ID).content(uri.stringValue(),false).closeTag(); - } - } - - private boolean simpleRenderHit(XMLWriter writer, Hit hit) throws IOException { - if (hit instanceof DefaultErrorHit) { - return simpleRenderDefaultErrorHit(writer, (DefaultErrorHit) hit); - } else if (hit instanceof GroupingListHit) { - return true; - } else { - return false; - } - } - - public static boolean simpleRenderDefaultErrorHit(XMLWriter writer, ErrorHit defaultErrorHit) throws IOException { - writer.openTag("errordetails"); - for (Iterator i = defaultErrorHit.errorIterator(); i.hasNext();) { - ErrorMessage error = (ErrorMessage) i.next(); - renderMessageDefaultErrorHit(writer, error); - } - writer.closeTag(); - return true; - } - - public static void renderMessageDefaultErrorHit(XMLWriter writer, ErrorMessage error) throws IOException { - writer.openTag("error"); - writer.attribute("source", error.getSource()); - writer.attribute("error", error.getMessage()); - writer.attribute("code", Integer.toString(error.getCode())); - writer.content(error.getDetailedMessage(), false); - if (error.getCause()!=null) { - writer.openTag("cause"); - writer.content("\n", true); - StringWriter stackTrace=new StringWriter(); - error.getCause().printStackTrace(new PrintWriter(stackTrace)); - writer.content(stackTrace.toString(), true); - writer.closeTag(); - } - writer.closeTag(); - } - - public static final class RenderingVisitor extends TraceVisitor { - - private static final String tag = "p"; - private final XMLWriter writer; - private long baseTime; - - public RenderingVisitor(XMLWriter writer,long baseTime) { - this.writer=writer; - this.baseTime=baseTime; - } - - @Override - public void entering(TraceNode node) { - if (node.isRoot()) return; - writer.openTag(tag); - } - - @Override - public void leaving(TraceNode node) { - if (node.isRoot()) return; - writer.closeTag(); - } - - @Override - public void visit(TraceNode node) { - if (node.isRoot()) return; - if (node.payload()==null) return; - - writer.openTag(tag); - if (node.timestamp()!=0) - writer.content(node.timestamp()-baseTime,false).content(" ms: ", false); - writer.content(node.payload().toString(),false); - writer.closeTag(); - } - - } - - private Result getResult() { - Result r; - try { - r = (Result) getResponse(); - } catch (ClassCastException e) { - throw new IllegalArgumentException( - "DefaultRenderer attempted used outside a search context, got a " - + getResponse().getClass().getName()); - } - return r; - } - - @Override - public void beginResponse(OutputStream stream) throws IOException { - Charset cs = Charset.forName(getRequestedEncoding(getResult().getQuery())); - CharsetEncoder encoder = cs.newEncoder(); - writer = wrapWriter(new ByteWriter(stream, encoder)); - - header(writer, getResult()); - if (getResult().hits().getError() != null || getResult().hits().getQuery().errors().size() > 0) { - error(writer, getResult()); - } - - if (getResult().getConcreteHitCount() == 0) { - emptyResult(writer, getResult()); - } - - if (getResult().getContext(false) != null) { - queryContext(writer, getResult().getContext(false), getResult().getQuery()); - } - - } - - /** Returns the encoding of the query, or the encoding given by the template if none is set */ - public final String getRequestedEncoding(Query query) { - String encoding = query.getModel().getEncoding(); - if (encoding != null) return encoding; - return getEncoding(); - } - - @Override - public void beginList(DataList<?> list) throws IOException { - if (getRecursionLevel() == 1) return; - - HitGroup hit = (HitGroup) list; - boolean renderedSimple = simpleRenderHit(writer, hit); - if (renderedSimple) return; - - renderHitGroup(writer, hit); - } - - @Override - public void data(Data data) throws IOException { - Hit hit = (Hit) data; - boolean renderedSimple = simpleRenderHit(writer, hit); - if (renderedSimple) return; - - renderSingularHit(writer, hit); - writer.closeTag(); - } - - @Override - public void endList(DataList<?> list) { - if (getRecursionLevel() > 1) - writer.closeTag(); - } - - @Override - public void endResponse() { - writer.closeTag(); - writer.close(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 7d60d7cf9ee..e8af150ce25 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -18,6 +18,7 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.document.json.JsonWriter; import com.yahoo.lang.MutableBoolean; +import com.yahoo.prelude.hitfield.HitField; import com.yahoo.processing.Response; import com.yahoo.processing.execution.Execution.Trace; import com.yahoo.processing.rendering.AsynchronousSectionedRenderer; @@ -473,7 +474,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeNumberField(RELEVANCE, hit.getRelevance().getScore()); - if (hit.types().size() > 0) { // TODO: Remove types rendering on Vespa 7 + if (hit.types().size() > 0) { generator.writeArrayFieldStart(TYPES); for (String t : hit.types()) { generator.writeString(t); diff --git a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java index 0f01261f6a4..783045babf4 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/RendererRegistry.java @@ -4,12 +4,9 @@ package com.yahoo.search.rendering; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.prelude.templates.PageTemplateSet; -import com.yahoo.prelude.templates.SearchRendererAdaptor; -import com.yahoo.prelude.templates.TiledTemplateSet; -import com.yahoo.prelude.templates.UserTemplate; import com.yahoo.processing.rendering.Renderer; import com.yahoo.search.Result; +import com.yahoo.search.pagetemplates.result.PageTemplatesXmlRenderer; import java.util.Collection; import java.util.Collections; @@ -23,14 +20,11 @@ import java.util.concurrent.Executor; */ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processing.rendering.Renderer<Result>> { - public static final ComponentId xmlRendererId = ComponentId.fromString("DefaultRenderer"); - private static final ComponentId newXmlRendererId = ComponentId.fromString("XmlRenderer"); + public static final ComponentId xmlRendererId = ComponentId.fromString("XmlRenderer"); + public static final ComponentId pageRendererId = ComponentId.fromString("PageTemplatesXmlRenderer"); public static final ComponentId jsonRendererId = ComponentId.fromString("JsonRenderer"); public static final ComponentId defaultRendererId = jsonRendererId; - private final ComponentId tiledRendererId; - private final ComponentId pageRendererId; - /** Creates a registry containing the built-in renderers only */ public RendererRegistry() { this(Collections.emptyList()); @@ -61,23 +55,19 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi register(jsonRenderer.getId(), jsonRenderer); // Add xml renderer - Renderer xmlRenderer = new DefaultRenderer(executor); + Renderer xmlRenderer = new XmlRenderer(executor); xmlRenderer.initId(xmlRendererId); register(xmlRenderer.getId(), xmlRenderer); - // Add new Vespa 7 xml renderer - Renderer newXmlRenderer = new XmlRenderer(executor); - newXmlRenderer.initId(newXmlRendererId); - register(newXmlRenderer.getId(), newXmlRenderer); + // Add page templates renderer + Renderer pageRenderer = new PageTemplatesXmlRenderer(executor); + pageRenderer.initId(pageRendererId); + register(pageRenderer.getId(), pageRenderer); // add application renderers for (Renderer renderer : renderers) register(renderer.getId(), renderer); - // add legacy "templates" converted to renderers // TODO: Remove on Vespa 7 - tiledRendererId = addTemplateSet(new TiledTemplateSet()); - pageRendererId = addTemplateSet(new PageTemplateSet()); - freeze(); } @@ -86,20 +76,9 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi // deconstruct the renderers which was created by this getRenderer(jsonRendererId.toSpecification()).deconstruct(); getRenderer(xmlRendererId.toSpecification()).deconstruct(); - getRenderer(newXmlRendererId.toSpecification()).deconstruct(); - getRenderer(tiledRendererId.toSpecification()).deconstruct(); getRenderer(pageRendererId.toSpecification()).deconstruct(); } - @SuppressWarnings({"deprecation", "unchecked"}) - private ComponentId addTemplateSet(UserTemplate<?> templateSet) { - Renderer renderer = new SearchRendererAdaptor(templateSet); - ComponentId rendererId = new ComponentId(templateSet.getName()); - renderer.initId(rendererId); - register(rendererId, renderer); - return rendererId; - } - /** * Returns the default JSON renderer * @@ -120,6 +99,7 @@ public final class RendererRegistry extends ComponentRegistry<com.yahoo.processi if (format == null || format.stringValue().equals("default")) return getDefaultRenderer(); if (format.stringValue().equals("json")) return getComponent(jsonRendererId); if (format.stringValue().equals("xml")) return getComponent(xmlRendererId); + if (format.stringValue().equals("page")) return getComponent(pageRendererId); com.yahoo.processing.rendering.Renderer<Result> renderer = getComponent(format); if (renderer == null) diff --git a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java index 5f99c531c95..5586fd2f996 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/XmlRenderer.java @@ -349,15 +349,12 @@ public final class XmlRenderer extends AsynchronousSectionedRenderer<Result> { } private Result getResult() { - Result r; try { - r = (Result) getResponse(); + return (Result) getResponse(); } catch (ClassCastException e) { - throw new IllegalArgumentException( - "XmlRenderer attempted used outside a search context, got a " - + getResponse().getClass().getName()); + throw new IllegalArgumentException("XmlRenderer attempted used outside a search context, got a " + + getResponse().getClass().getName()); } - return r; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/result/Coverage.java b/container-search/src/main/java/com/yahoo/search/result/Coverage.java index 7bc49fdb0b1..25829b70b5e 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Coverage.java +++ b/container-search/src/main/java/com/yahoo/search/result/Coverage.java @@ -23,20 +23,6 @@ public class Coverage extends com.yahoo.container.handler.Coverage { super(docs, active, nodes, resultSets); } - // TODO: Remove on Vespa 7 - /** @deprecated don't send a "full" boolean */ - @Deprecated // OK - public Coverage(long docs, int nodes, boolean full) { - this(docs, nodes, full, 1); - } - - // TODO: Remove on Vespa 7 - /** @deprecated don't send a "full" boolean */ - @Deprecated // OK - public Coverage(long docs, int nodes, boolean full, int resultSets) { - super(docs, nodes, full, resultSets); - } - /** * Will set number of documents present in ideal state * diff --git a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java index 54b83544b1a..d8ec768325a 100644 --- a/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/DefaultErrorHit.java @@ -58,18 +58,6 @@ public class DefaultErrorHit extends Hit implements ErrorHit, Cloneable { } /** - * Returns the main error of this result, never null. - * - * @deprecated use {@link #errors()} - */ - @Override - @Deprecated // OK - // TODO: Remove on Vespa 7 - public ErrorMessage getMainError() { - return errors.get(0); - } - - /** * This is basically a way of making a list simulate a set. */ private void removeAndAdd(ErrorMessage error) { diff --git a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java index 091c8d67793..9be80372ba9 100644 --- a/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java +++ b/container-search/src/main/java/com/yahoo/search/result/ErrorHit.java @@ -14,15 +14,6 @@ public interface ErrorHit extends Cloneable { void setSource(String source); - /** - * Returns the main error of this result, never null - * - * @deprecated use errors().iterator().next() - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - ErrorMessage getMainError(); - /** * Adds an error to this. This may change the main error * and/or the list of detailed errors diff --git a/container-search/src/main/java/com/yahoo/search/result/Hit.java b/container-search/src/main/java/com/yahoo/search/result/Hit.java index c5132549d05..db5b3e69803 100644 --- a/container-search/src/main/java/com/yahoo/search/result/Hit.java +++ b/container-search/src/main/java/com/yahoo/search/result/Hit.java @@ -106,9 +106,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** If this is true, then this hit will not be counted as a concrete hit */ private boolean auxiliary = false; - /** - * The hit field used to store rank features. TODO: Remove on Vespa 7 - */ + /** The hit field used to store rank features */ public static final String RANKFEATURES_FIELD = "rankfeatures"; public static final String SDDOCNAME_FIELD = "sddocname"; @@ -420,7 +418,7 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi */ public void forEachFieldAsRaw(RawUtf8Consumer consumer) { if (fields == null) return; - fields.forEach(consumer); // No utf-8 fields available in Hit + fields.forEach(consumer); } /** Returns the fields of this as a read-only map. This is more costly than fieldIterator() */ @@ -478,20 +476,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return fields != null && ! fields.isEmpty(); } - /** - * Changes the key under which a value is found. This is useful because it allows keys to be changed - * without accessing the value (which may be lazily created). - * - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void changeFieldKey(String oldKey, String newKey) { - Map<String,Object> fieldMap = getFieldMap(); - Object value = fieldMap.remove(oldKey); - fieldMap.put(newKey, value); - } - private Map<String, Object> getFieldMap() { return getFieldMap(2); } @@ -515,42 +499,16 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return unmodifiableFieldMap; } - /** Generate a HitField from a field if the field exists */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public HitField buildHitField(String key) { - return buildHitField(key, false); - } - - /** Generate a HitField from a field if the field exists */ - @SuppressWarnings("deprecation") - // TODO: Remove on Vespa 7 - @Deprecated // OK public HitField buildHitField(String key, boolean forceNoPreTokenize) { - return buildHitField(key, forceNoPreTokenize, false); - } - - // TODO: Remove third parameter on Vespa 7 (set always true) - @Deprecated // OK - public HitField buildHitField(String key, boolean forceNoPreTokenize, boolean forceStringHandling) { Object o = getField(key); if (o == null) return null; if (o instanceof HitField) return (HitField)o; HitField h; - if (forceNoPreTokenize) { - if (o instanceof XMLString && !forceStringHandling) { - h = new HitField(key, (XMLString) o, false); - } else { - h = new HitField(key, o.toString(), false); - } - } else { - if (o instanceof XMLString && !forceStringHandling) { - h = new HitField(key, (XMLString) o); - } else { - h = new HitField(key, o.toString()); - } - } + if (forceNoPreTokenize) + h = new HitField(key, o.toString(), false); + else + h = new HitField(key, o.toString()); h.setOriginal(o); getFieldMap().put(key, h); return h; @@ -559,34 +517,19 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi /** Returns the types of this as a modifiable set. Modifications to this set are directly reflected in this hit */ public Set<String> types() { return types; } - /** @deprecated do not use */ - // TODO: FRemove on Vespa 7 - @Deprecated // OK - public String getTypeString() { - return types().stream().collect(Collectors.joining(" ")); - } - /** * Returns the add number, assigned when adding the hit to a Result. * * Used to order equal relevant hit by add order. -1 if this hit * has never been added to a result. - * - * @deprecated do not use */ - // TODO: Make package private on Vespa 7 - @Deprecated // OK - public int getAddNumber() { return addNumber; } + int getAddNumber() { return addNumber; } /** * Sets the add number, assigned when adding the hit to a Result, * used to order equal relevant hit by add order. - * - * @deprecated do not use */ - // TODO: Make package private on Vespa 7 - @Deprecated // OK - public void setAddNumber(int addNumber) { this.addNumber = addNumber; } + void setAddNumber(int addNumber) { this.addNumber = addNumber; } /** * Returns whether this is a concrete hit, containing content of the requested @@ -610,16 +553,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi public void setAuxiliary(boolean auxiliary) { this.auxiliary = auxiliary; } - /** @deprecated do not use */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public int getSourceNumber() { return sourceNumber; } - - /** @deprecated do not use */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setSourceNumber(int number) { this.sourceNumber = number; } - /** Returns the query which produced this hit, or null if not known */ public Query getQuery() { return query; } @@ -633,52 +566,6 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi } } - /** - * Returns a field of this hit XML escaped and without token - * delimiters. - * - * @deprecated do not use - * @return a field of this hit, or null if the property is not set - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public String getFieldXML(String key) { - Object p = getField(key); - - if (p == null) { - return null; - } else if (p instanceof HitField) { - return ((HitField)p).quotedContent(false); - } else if (p instanceof StructuredData || p instanceof XMLString || p instanceof JSONString) { - return p.toString(); - } else { - return XML.xmlEscape(p.toString(), false, '\u001f'); - } - } - - /** - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public String getUnboldedField(String key, boolean escape) { - Object p = getField(key); - - if (p == null) { - return null; - } else if (p instanceof HitField) { - return ((HitField) p).bareContent(escape, false); - } else if (p instanceof StructuredData) { - return p.toString(); - } else if (p instanceof XMLString || p instanceof JSONString) { - return p.toString(); - } else if (escape) { - return XML.xmlEscape(p.toString(), false, '\u001f'); - } else { - return stripCharacter('\u001F', p.toString()); - } - } - /** Attach some data to this hit for this searcher */ public void setSearcherSpecificMetaData(Searcher searcher, Object data) { if (searcherSpecificMetaData == null) { @@ -703,60 +590,21 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi return searcherSpecificMetaData != null ? searcherSpecificMetaData.get(searcher) : null; } - /** - * Internal - do not use - * - * @param filled the backing set - */ - // TODO: Make package private on Vespa 7 - protected final void setFilledInternal(Set<String> filled) { + final void setFilledInternal(Set<String> filled) { this.filled = filled; unmodifiableFilled = (filled != null) ? Collections.unmodifiableSet(filled) : null; } /** - * For internal use only. * Gives access to the modifiable backing set of filled summaries. * This set might be unmodifiable if the size is less than or equal to 1 * * @return the set of filled summaries. */ - // TODO: Make package private on Vespa 7 - protected final Set<String> getFilledInternal() { + final Set<String> getFilledInternal() { return filled; } - /** - * @deprecated do not use - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public static String stripCharacter(char strip, String toStripFrom) { - StringBuilder builder = null; - - int lastBadChar = 0; - for (int i = 0; i < toStripFrom.length(); i++) { - if (toStripFrom.charAt(i) == strip) { - if (builder == null) { - builder = new StringBuilder(toStripFrom.length()); - } - - builder.append(toStripFrom, lastBadChar, i); - lastBadChar = i + 1; - } - } - - if (builder == null) { - return toStripFrom; - } else { - if (lastBadChar < toStripFrom.length()) { - builder.append(toStripFrom, lastBadChar, toStripFrom.length()); - } - - return builder.toString(); - } - } - /** Releases the resources held by this, making it irreversibly unusable */ protected void close() { query = null; @@ -832,8 +680,8 @@ public class Hit extends ListenableFreezableClass implements Data, Comparable<Hi * * @param fieldName the name of the field * @param utf8Data raw utf-8 data. The reciver <b>must not</b> modify this data - * @param offset the start index of the data to accept into the utf8Data array - * @param length the length of the data to accept into the utf8Data array + * @param offset the start index in the utf8Data array of the data to accept + * @param length the length starting from offset in the utf8Data array of the data to accept */ void accept(String fieldName, byte[] utf8Data, int offset, int length); diff --git a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java index 3d5cbab2865..af89220e504 100644 --- a/container-search/src/main/java/com/yahoo/search/result/HitGroup.java +++ b/container-search/src/main/java/com/yahoo/search/result/HitGroup.java @@ -383,17 +383,6 @@ public class HitGroup extends Hit implements DataList<Hit>, Cloneable, Iterable< return hit; } - /** - * Sets the main error of this result - * - * @deprecated prefer addError to add some error information. - */ - // TODO: Remove on Vespa 7 - @Deprecated // OK - public void setError(ErrorMessage error) { - addError(error); - } - /** Adds an error to this result */ public void addError(ErrorMessage error) { getError(); // update the list of errors diff --git a/container-search/src/main/java/com/yahoo/search/result/PositionsData.java b/container-search/src/main/java/com/yahoo/search/result/PositionsData.java index a07eaa2438d..483849a5435 100644 --- a/container-search/src/main/java/com/yahoo/search/result/PositionsData.java +++ b/container-search/src/main/java/com/yahoo/search/result/PositionsData.java @@ -17,8 +17,8 @@ public class PositionsData implements Inspectable, JsonProducer, XmlProducer { public PositionsData(Inspector value) { this.value = value; - if (value.type() != Type.ARRAY) { - throw new IllegalArgumentException("PositionsData expects an array of positions, got: "+value); + if (value.type() != Type.OBJECT && value.type() != Type.ARRAY) { + throw new IllegalArgumentException("PositionsData expects a position or an array of positions, got: "+value); } } @@ -38,27 +38,35 @@ public class PositionsData implements Inspectable, JsonProducer, XmlProducer { @Override public StringBuilder writeXML(StringBuilder target) { - for (int i = 0; i < value.entryCount(); i++) { - Inspector pos = value.entry(i); - target.append("<position "); - for (java.util.Map.Entry<String, Inspector> entry : pos.fields()) { - Inspector v = entry.getValue(); - if (v.type() == Type.STRING) { - target.append(entry.getKey()); - target.append("=\""); - target.append(entry.getValue().asString()); - target.append("\" "); - } - if (v.type() == Type.LONG) { - target.append(entry.getKey()); - target.append("=\""); - target.append(entry.getValue().asLong()); - target.append("\" "); - } + if (value.type() == Type.OBJECT) { + writeXML(value.inspect(), target); + } else { + for (int i = 0; i < value.entryCount(); i++) { + Inspector pos = value.entry(i); + writeXML(pos, target); } - target.append("/>"); } return target; } + private static void writeXML(Inspector pos, StringBuilder target) { + target.append("<position "); + for (java.util.Map.Entry<String, Inspector> entry : pos.fields()) { + Inspector v = entry.getValue(); + if (v.type() == Type.STRING) { + target.append(entry.getKey()); + target.append("=\""); + target.append(entry.getValue().asString()); + target.append("\" "); + } + if (v.type() == Type.LONG) { + target.append(entry.getKey()); + target.append("=\""); + target.append(entry.getValue().asLong()); + target.append("\" "); + } + } + target.append("/>"); + } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/Templating.java b/container-search/src/main/java/com/yahoo/search/result/Templating.java deleted file mode 100644 index a8b1eedf528..00000000000 --- a/container-search/src/main/java/com/yahoo/search/result/Templating.java +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.result; - -import java.util.Map; - -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.prelude.templates.SearchRendererAdaptor; -import com.yahoo.prelude.templates.TemplateSet; -import com.yahoo.prelude.templates.UserTemplate; -import com.yahoo.processing.rendering.Renderer; -import com.yahoo.search.Result; -import com.yahoo.search.query.Presentation; - -/** - * Helper methods and data store for result attributes geared towards result - * rendering and presentation. - * - * @author Steinar Knutsen - * @deprecated do not use - */ -@Deprecated // OK (But wait for deprecated handlers in vespaclient-container-plugin to be removed) -// TODO: Remove on Vespa 7 -public class Templating { - - private final Result result; - private Renderer<Result> renderer; - - public Templating(Result result) { - super(); - this.result = result; - } - - /** - * Returns The first hit presented in the result as an index into the global - * list of all hits generated by the user query. - */ - public int getFirstHitNo() { - return result.getQuery().getOffset() + 1; - } - - /** - * Returns the first hit of the next result page, 0 if there aren't any more - * hits available - */ - public long getNextFirstHitNo() { - if (result.getQuery().getHits() > result.getConcreteHitCount()) { - return 0; - } - - return Math.min(getLastHitNo() + 1, result.getTotalHitCount()); - } - - /** - * Returns the first hit of the next result page, 0 if there aren't any more - * hits available - */ - public long getNextLastHitNo() { - if (result.getQuery().getHits() > result.getConcreteHitCount()) { - return 0; - } - - return Math.min(getLastHitNo() + result.getConcreteHitCount(), result.getTotalHitCount()); - } - - /** - * Returns the number of the last result of the current hit page. - */ - public int getLastHitNo() { - return getFirstHitNo() + result.getConcreteHitCount() - 1; - } - - /** - * The first hit presented on the previous result page as an index into the - * global list of all hits generated by the user query - */ - public int getPrevFirstHitNo() { - return Math.max(getFirstHitNo() - result.getQuery().getHits(), 1); - } - - /** - * The last hit presented on the previous result page as an index into the - * global list of all hits generated by the user query - */ - public int getPrevLastHitNo() { - return Math.max(getFirstHitNo() - 1, 0); - } - - /** - * An URL that may be used to obtain the next result page. - */ - public String getNextResultURL() { - HttpRequest request = result.getQuery().getHttpRequest(); - StringBuilder nextURL = new StringBuilder(); - - nextURL.append(getPath(request)).append("?"); - parametersExceptOffset(request, nextURL); - - int offset = getLastHitNo(); - - nextURL.append("&").append("offset=").append(Integer.toString(offset)); - return nextURL.toString(); - } - - /** - * An URL that may be used to obtain the previous result page. - */ - public String getPreviousResultURL() { - HttpRequest request = result.getQuery().getHttpRequest(); - StringBuilder prevURL = new StringBuilder(); - - prevURL.append(getPath(request)).append("?"); - parametersExceptOffset(request, prevURL); - int offset = getPrevFirstHitNo() - 1; - prevURL.append("&").append("offset=").append(Integer.toString(offset)); - return prevURL.toString(); - } - - public String getCurrentResultURL() { - HttpRequest request = result.getQuery().getHttpRequest(); - StringBuilder thisURL = new StringBuilder(); - - thisURL.append(getPath(request)).append("?"); - parameters(request, thisURL); - return thisURL.toString(); - } - - private String getPath(HttpRequest request) { - String path = request.getUri().getPath(); - if (path == null) { - path = ""; - } - return path; - } - - private void parametersExceptOffset(HttpRequest request, StringBuilder nextURL) { - int startLength = nextURL.length(); - for (Map.Entry<String, String> property : request.propertyMap().entrySet()) { - if (property.getKey().equals("offset")) continue; - - if (nextURL.length() > startLength) - nextURL.append("&"); - nextURL.append(property.getKey()).append("=").append(property.getValue()); - } - } - - private void parameters(HttpRequest request, StringBuilder nextURL) { - int startLength = nextURL.length(); - for (Map.Entry<String, String> property : request.propertyMap().entrySet()) { - if (nextURL.length() > startLength) - nextURL.append("&"); - nextURL.append(property.getKey()).append("=").append(property.getValue()); - } - } - - /** - * Returns the templates which will render the result. This is never null. - * If default rendering is used, it is a TemplateSet containing no - * templates. - * - * @deprecated use a renderer instead - */ - @SuppressWarnings("rawtypes") - // TODO: Remove on Vespa 7 - @Deprecated // OK - public UserTemplate getTemplates() { - if (renderer == null) { - return TemplateSet.getDefault(); - } else if (renderer instanceof SearchRendererAdaptor) { - return ((SearchRendererAdaptor) renderer).getAdaptee(); - } else { - throw new RuntimeException( - "Please use getTemplate() instead of getTemplates() when using the new template api."); - } - } - - /** - * Sets the template set which should render this result set - * - * @param templates - * the templates which should render this result, or null to - * use the default xml rendering - */ - @SuppressWarnings("deprecation") - public void setTemplates(@SuppressWarnings("rawtypes") UserTemplate templates) { - if (templates == null) { - setTemplates(TemplateSet.getDefault()); - } else { - setRenderer(new SearchRendererAdaptor(templates)); - } - } - - /** - * @deprecated since 5.1.21, use {@link Presentation#getRenderer()} - */ - public Renderer<Result> getRenderer() { - return renderer; - } - - /** - * @deprecated since 5.1.21, use {@link Presentation#setRenderer(com.yahoo.component.ComponentSpecification)} - */ - public void setRenderer(Renderer<Result> renderer) { - this.renderer = renderer; - } - - /** - * For internal use only. - */ - public boolean usesDefaultTemplate() { - return renderer == null || - (renderer instanceof SearchRendererAdaptor && - ((SearchRendererAdaptor) renderer).getAdaptee().isDefaultTemplateSet()); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java deleted file mode 100644 index a679b17b6fc..00000000000 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.searchchain.model.federation; - -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import net.jcip.annotations.Immutable; - -import com.yahoo.search.federation.http.HTTPProviderSearcher; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Specifies how a http provider is to be set up. - * - * @author Tony Vaagenes - * @deprecated - */ -@Immutable -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class HttpProviderSpec { - - public enum Type { - vespa(com.yahoo.search.federation.vespa.VespaSearcher.class); - - Type(Class<? extends HTTPProviderSearcher> searcherClass) { - className = searcherClass.getName(); - } - - final String className; - } - - // The default connection parameter values come from the config definition - public static class ConnectionParameters { - public final Double readTimeout; - public final Double connectionTimeout; - public final Double connectionPoolTimeout; - public final Integer retries; - - public ConnectionParameters(Double readTimeout, Double connectionTimeout, - Double connectionPoolTimeout, Integer retries) { - this.readTimeout = readTimeout; - this.connectionTimeout = connectionTimeout; - this.connectionPoolTimeout = connectionPoolTimeout; - this.retries = retries; - } - } - - public static class Node { - public final String host; - public final int port; - - public Node(String host, int port) { - this.host = host; - this.port = port; - } - - @Override - public String toString() { - return "Node{" + - "host='" + host + '\'' + - ", port=" + port + - '}'; - } - } - - public final ConnectionParameters connectionParameters; - - public final Integer cacheSizeMB; - - public final String path; - public final List<Node> nodes; - public final String ycaApplicationId; - public final Integer ycaCertificateTtl; - public final Integer ycaRetryWait; - public final Node ycaProxy; - - public static BundleInstantiationSpecification toBundleInstantiationSpecification(Type type) { - return BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(type.className, null); - } - - public static boolean includesType(String typeString) { - for (Type type : Type.values()) { - if (type.name().equals(typeString)) { - return true; - } - } - return false; - } - - public HttpProviderSpec(Double cacheWeight, - String path, - List<Node> nodes, - String ycaApplicationId, - Integer ycaCertificateTtl, - Integer ycaRetryWait, - Node ycaProxy, - Integer cacheSizeMB, - ConnectionParameters connectionParameters) { - - this.path = path; - this.nodes = unmodifiable(nodes); - this.ycaApplicationId = ycaApplicationId; - this.ycaProxy = ycaProxy; - this.ycaCertificateTtl = ycaCertificateTtl; - this.ycaRetryWait = ycaRetryWait; - this.cacheSizeMB = cacheSizeMB; - - this.connectionParameters = connectionParameters; - } - - private List<HttpProviderSpec.Node> unmodifiable(List<HttpProviderSpec.Node> nodes) { - return nodes == null ? - Collections.emptyList() : - Collections.unmodifiableList(new ArrayList<>(nodes)); - } -} |