diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo/search/federation/http')
13 files changed, 0 insertions, 2352 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java deleted file mode 100644 index 1607d108722..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.Collections; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - - -/** - * Superclass for http client searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPClientSearcher extends HTTPClientSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPClientSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id, ConfiguredSearcherHelper.toConnectionList(providerConfig), new HTTPParameters(providerConfig), manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPClientSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Forwards to the next in chain fill(result,summaryName) */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - execution.fill(result,summaryName); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java deleted file mode 100644 index 9f99a790df0..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import java.util.Collections; - - -/** - * Superclass for http provider searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPProviderSearcher extends HTTPProviderSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - } - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,HTTPParameters parameters, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),parameters, manager); - } - - /** - * Override this to provider multi-phase result filling towards a backend. - * This default implementation does nothing. - */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java deleted file mode 100644 index 0fa235709f9..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * Some static helper classes for configured*Searcher classes - * - * @author bratseth - */ -// TODO: Remove on Vespa 7 -class ConfiguredSearcherHelper { - - /** No instantiation */ - private ConfiguredSearcherHelper() { } - - public static List<Connection> toConnectionList(ProviderConfig providerConfig) { - List<Connection> connections=new ArrayList<>(); - for(ProviderConfig.Node node : providerConfig.node()) { - connections.add(new Connection(node.host(), node.port())); - } - return connections; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java b/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java deleted file mode 100644 index 4f624a12547..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Represents a connection to a particular node (host/port). - * Right now this is just a container of connection parameters, but might be extended to - * contain an open connection later. - * The host and port state is immutable. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class Connection { - - private String host; - private int port; - - public Connection(String host,int port) { - this.host=host; - this.port=port; - } - - public String getHost() { return host; } - - public int getPort() { return port; } - - public String toString() { - return "http connection '" + host + ":" + port + "'"; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java deleted file mode 100644 index 0407ddeca1b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import org.apache.http.HttpEntity; -import org.apache.http.entity.HttpEntityWrapper; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.zip.GZIPInputStream; - -/** - * Used by HTTPSearcher when talking to services returning compressed content. - * - * @author Mainak Mandal - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class GzipDecompressingEntity extends HttpEntityWrapper { - - private static class Resources { - - byte [] buffer; - int total; - - Resources() { - total = 0; - buffer = new byte[65536]; - } - void drain(InputStream zipStream) throws IOException { - int numRead = zipStream.read(buffer, total, buffer.length); - while (numRead != -1) { - total += numRead; - if ((total + 65536) > buffer.length) { - buffer = Arrays.copyOf(buffer, buffer.length + numRead); - } - numRead = zipStream.read(buffer, total, buffer.length - total); - } - } - - } - - private final Resources resources = new Resources(); - - public GzipDecompressingEntity(final HttpEntity entity) throws IllegalStateException, IOException { - super(entity); - GZIPInputStream gz = new GZIPInputStream(entity.getContent()); - InputStream zipStream = new BufferedInputStream(gz); - try { - resources.drain(zipStream); - } catch (IOException e) { - throw e; - } finally { - zipStream.close(); - } - } - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - - final ByteBuffer buff = ByteBuffer.wrap(resources.buffer, 0, resources.total); - return new InputStream() { - - @Override - public int available() throws IOException { - return buff.remaining(); - } - - @Override - public int read() throws IOException { - if (buff.hasRemaining()) - return buff.get() & 0xFF; - - return -1; - } - - @Override - public int read(byte[] b) throws IOException { - if (!buff.hasRemaining()) - return -1; - - int len = b.length; - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, 0, len); - return len; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, off, len); - return len; - } - - @Override - public long skip(long n) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (n > buff.remaining()) - n = buff.remaining(); - - buff.position(buff.position() + (int) n); - return n; - } - }; - } - - @Override - public long getContentLength() { - return resources.total; - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - outstream.write(resources.buffer, 0, resources.total); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java deleted file mode 100644 index ee093adc6fc..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -/** - * A utility parent for searchers which gets data from web services which is incorporated into the query. - * This searcher will take care of implementing the search method while the extending class implements - * {@link #getQueryMap} and {@link #handleResponse} to create the http request and handle the response, respectively. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, is this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPClientSearcher extends HTTPSearcher { - - static final CompoundName REQUEST_META_CARRIER = new CompoundName("com.yahoo.search.federation.http.HTTPClientSearcher_requestMeta"); - - protected final static Logger log = Logger.getLogger(HTTPClientSearcher.class.getName()); - - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param path the path portion of the url to be used - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics) { - super(id, connections, path, statistics); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics, - CertificateStore certificateStore) { - super(id, connections, path, statistics, certificateStore); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, Statistics statistics) { - super(id, connections, parameters, statistics); - } - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - * @param certificateStore the certificate store to use to pass certificates in requests - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - } - - /** Overridden to avoid interfering with errors from nested searchers, which is inappropriate for a <i>client</i> */ - @Override - public Result robustSearch(Query query, Execution execution, Connection connection) { - return search(query,execution,connection); - } - - /** Implements a search towards the connection chosen by the cluster searcher for this query */ - @Override - public Result search(Query query, Execution execution, Connection connection) { - Hit requestMeta = doHttpRequest(query, connection); - Result result = execution.search(query); - result.hits().add(requestMeta); - return result; - } - - private Hit doHttpRequest(Query query, Connection connection) { - URI uri; - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - query.properties().set(REQUEST_META_CARRIER, requestMeta); - - query.trace("Created request information hit",false,9); - try { - uri = getURI(query, connection); - } catch (MalformedURLException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } catch (URISyntaxException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } - - HttpEntity entity; - try { - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - entity = getEntity(uri, requestMeta, query); - } catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this + " using " + connection + " for " + - query + ": " + Exceptions.toMessageString(e))); - return requestMeta; - } catch (TimeoutException e) { - query.errors().add(ErrorMessage.createTimeout("HTTP traffic timed out in " - + this + " for " + query + ": " + e.getMessage())); - return requestMeta; - } - if (entity==null) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "No result from connecting to HTTP backend in " + this + " using " + connection + " for " + query)); - return requestMeta; - } - - try { - query = handleResponse(entity,query); - } - catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - } finally { - cleanupHttpEntity(entity); - } - return requestMeta; - } - - /** Overrides to pass the query on to the next searcher */ - @Override - public Result search(Query query, Execution execution, ErrorMessage error) { - query.errors().add(error); - return execution.search(query); - } - - /** Do nothing on fill in client searchers */ - @Override - public void fill(Result result,String summaryClass,Execution execution,Connection connection) { - } - - /** - * Convenience hook for unmarshalling the response and adding the information to the query. - * Implement this or <code>handleResponse(entity,query)</code> in any subclass. - * This default implementation throws an exception. - * - * @param inputStream the stream containing the data from the http service - * @param contentLength the length of the content in the stream in bytes, or a negative number if not known - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - throw new UnsupportedOperationException("handleResponse must be implemented by " + this); - } - - /** - * Unmarshals the response and adds the resulting data to the given query. - * This default implementation calls - * <code>return handleResponse(entity.getContent(), entity.getContentLength(), query);</code> - * (and does some detailed query tracing). - * - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(HttpEntity entity, Query query) throws IOException { - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query = handleResponse(entity.getContent(), len, query); - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - return query; - } - - /** Never retry individual queries to clients for now */ - @Override - protected boolean shouldRetry(Query query,Result result) { return false; } - - /** - * numHits and offset should not be part of the cache key as cache supports - * partial read/write that is only one cache entry is maintained per query - * irrespective of the offset and numhits. - */ - public abstract Map<String, String> getCacheKey(Query q); - - /** - * Adds all key-values starting by "service." + getClientName() in query.properties(). - * Returns the empty map if {@link #getServiceName} is not overridden. - */ - @Override - public Map<String,String> getQueryMap(Query query) { - LinkedHashMap<String, String> queryMap=new LinkedHashMap<>(); - if (getServiceName().isEmpty()) return queryMap; - - for (Map.Entry<String,Object> objectProperty : query.properties().listProperties("service." + getServiceName()).entrySet()) // TODO: Make more efficient using CompoundName - queryMap.put(objectProperty.getKey(),objectProperty.getValue().toString()); - return queryMap; - } - - /** - * Override this to return the name of the service this is a client of. - * This is used to look up service specific properties as service.getServiceName.serviceSpecificProperty. - * This default implementation returns "", which means service specific parameters will not be used. - */ - protected String getServiceName() { return ""; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java deleted file mode 100644 index a7b32678c1a..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.base.Preconditions; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnPerRouteBean; -import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.params.HttpParams; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * A set of parameters for talking to an http backend - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public final class HTTPParameters { - - public static final String RETRIES = "com.yahoo.search.federation.http.retries"; - - private boolean frozen=false; - - // All timing parameters below are in milliseconds - /** The url request path portion */ - private String path="/"; - private int connectionTimeout=2000; - private int readTimeout=5000; - private boolean persistentConnections=true; - private boolean enableProxy = false; - private String proxyHost = "localhost"; - private int proxyPort = 1080; - private String method = "GET"; - private String schema = "http"; - private String inputEncoding = "utf-8"; - private String outputEncoding = "utf-8"; - private int maxTotalConnections=10000; - private int maxConnectionsPerRoute=10000; - private int socketBufferSizeBytes=-1; - private int retries = 1; - private int configuredReadTimeout = -1; - private int configuredConnectionTimeout = -1; - private int connectionPoolTimeout = -1; - private String certificateProxy = null; - private int certificatePort = 0; - private String certificateApplicationId = null; - private boolean certificateUseProxy = false; - private long certificateTtl = 0L; - private long certificateRetry = 0L; - - private PingOption.Enum pingOption = PingOption.NORMAL; - - - private boolean followRedirects = true; - - public HTTPParameters() {} - - public HTTPParameters(String path) { - setPath(path); - } - - public HTTPParameters(ProviderConfig providerConfig) { - configuredReadTimeout = (int) (providerConfig.readTimeout() * 1000.0d); - configuredConnectionTimeout = (int) (providerConfig.connectionTimeout() * 1000.0d); - connectionPoolTimeout = (int) (providerConfig.connectionPoolTimeout() * 1000.0d); - retries = providerConfig.retries(); - setPath(providerConfig.path()); - certificateUseProxy = providerConfig.yca().useProxy(); - if (certificateUseProxy) { - certificateProxy = providerConfig.yca().host(); - certificatePort = providerConfig.yca().port(); - } - certificateApplicationId = providerConfig.yca().applicationId(); - certificateTtl = providerConfig.yca().ttl() * 1000L; - certificateRetry = providerConfig.yca().retry() * 1000L; - followRedirects = providerConfig.followRedirects(); - pingOption = providerConfig.pingOption(); - } - - /** - * Set the url path to use in queries to this. If the argument is null or empty the path is set to "/". - * If a leading "/" is missing, it is added automatically. - */ - public final void setPath(String path) { - if (path==null || path.isEmpty()) path="/"; - - if (! path.startsWith("/")) - path="/" + path; - this.path = path; - } - - public PingOption.Enum getPingOption() { - return pingOption; - } - - public void setPingOption(PingOption.Enum pingOption) { - Preconditions.checkNotNull(pingOption); - ensureNotFrozen(); - this.pingOption = pingOption; - } - - /** Returns the url path. Default is "/". */ - public String getPath() { return path; } - - public boolean getFollowRedirects() { - return followRedirects; - } - - public void setFollowRedirects(boolean followRedirects) { - ensureNotFrozen(); - this.followRedirects = followRedirects; - } - - - public void setConnectionTimeout(int connectionTimeout) { - ensureNotFrozen(); - this.connectionTimeout=connectionTimeout; - } - - /** Returns the connection timeout in milliseconds. Default is 2000. */ - public int getConnectionTimeout() { return connectionTimeout; } - - public void setReadTimeout(int readTimeout) { - ensureNotFrozen(); - this.readTimeout=readTimeout; - } - - /** Returns the read timeout in milliseconds. Default is 5000. */ - public int getReadTimeout() { return readTimeout; } - - /** - * <b>Note: This is currently largely a noop: Connections are reused even when this is set to true. - * The setting will change from sharing connections between threads to only reusing it within a thread - * but it is still reused.</b> - */ - public void setPersistentConnections(boolean persistentConnections) { - ensureNotFrozen(); - this.persistentConnections=persistentConnections; - } - - /** Returns whether this should use persistent connections. Default is true. */ - public boolean getPersistentConnections() { return persistentConnections; } - - /** Returns whether proxying should be enabled. Default is false. */ - public boolean getEnableProxy() { return enableProxy; } - - public void setEnableProxy(boolean enableProxy ) { - ensureNotFrozen(); - this.enableProxy=enableProxy; - } - - /** Returns the proxy type to use (if enabled). Default is "http". */ - public String getProxyType() { - return "http"; - } - - public void setProxyHost(String proxyHost) { - ensureNotFrozen(); - this.proxyHost=proxyHost; - } - - /** Returns the proxy host to use (if enabled). Default is "localhost". */ - public String getProxyHost() { return proxyHost; } - - public void setProxyPort(int proxyPort) { - ensureNotFrozen(); - this.proxyPort=proxyPort; - } - - /** Returns the proxy port to use (if enabled). Default is 1080. */ - public int getProxyPort() { return proxyPort; } - - public void setMethod(String method) { - ensureNotFrozen(); - this.method=method; - } - - /** Returns the http method to use. Default is "GET". */ - public String getMethod() { return method; } - - public void setSchema(String schema) { - ensureNotFrozen(); - this.schema=schema; - } - - /** Returns the schema to use. Default is "http". */ - public String getSchema() { return schema; } - - public void setInputEncoding(String inputEncoding) { - ensureNotFrozen(); - this.inputEncoding=inputEncoding; - } - - /** Returns the input encoding. Default is "utf-8". */ - public String getInputEncoding() { return inputEncoding; } - - public void setOutputEncoding(String outputEncoding) { - ensureNotFrozen(); - this.outputEncoding=outputEncoding; - } - - /** Returns the output encoding. Default is "utf-8". */ - public String getOutputEncoding() { return outputEncoding; } - - /** Make this unmodifiable. Note that any thread synchronization must be done outside this object. */ - public void freeze() { - frozen=true; - } - - private void ensureNotFrozen() { - if (frozen) throw new IllegalStateException("Cannot modify frozen " + this); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams() { - return toHttpParams(connectionTimeout, readTimeout); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams(int connectionTimeout, int readTimeout) { - HttpParams params = new BasicHttpParams(); - // force use of configured value if available - if (configuredConnectionTimeout > 0) { - HttpConnectionParams.setConnectionTimeout(params, configuredConnectionTimeout); - } else { - HttpConnectionParams.setConnectionTimeout(params, connectionTimeout); - } - if (configuredReadTimeout > 0) { - HttpConnectionParams.setSoTimeout(params, configuredReadTimeout); - } else { - HttpConnectionParams.setSoTimeout(params, readTimeout); - } - if (socketBufferSizeBytes > 0) { - HttpConnectionParams.setSocketBufferSize(params, socketBufferSizeBytes); - } - if (connectionPoolTimeout > 0) { - ConnManagerParams.setTimeout(params, connectionPoolTimeout); - } - ConnManagerParams.setMaxTotalConnections(params, maxTotalConnections); - ConnManagerParams.setMaxConnectionsPerRoute(params, new ConnPerRouteBean(maxConnectionsPerRoute)); - if (retries >= 0) { - params.setIntParameter(RETRIES, retries); - } - params.setParameter("http.protocol.handle-redirects", followRedirects); - return params; - } - - public int getMaxTotalConnections() { - return maxTotalConnections; - } - - public void setMaxTotalConnections(int maxTotalConnections) { - ensureNotFrozen(); - this.maxTotalConnections = maxTotalConnections; - } - - public int getMaxConnectionsPerRoute() { - return maxConnectionsPerRoute; - } - - public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) { - ensureNotFrozen(); - this.maxConnectionsPerRoute = maxConnectionsPerRoute; - } - - public int getSocketBufferSizeBytes() { - return socketBufferSizeBytes; - } - - public void setSocketBufferSizeBytes(int socketBufferSizeBytes) { - ensureNotFrozen(); - this.socketBufferSizeBytes = socketBufferSizeBytes; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - ensureNotFrozen(); - this.retries = retries; - } - - public String getYcaProxy() { - return certificateProxy; - } - - public int getYcaPort() { - return certificatePort; - } - - public String getYcaApplicationId() { - return certificateApplicationId; - } - - public boolean getYcaUseProxy() { - return certificateUseProxy; - } - - public long getYcaTtl() { - return certificateTtl; - } - - public long getYcaRetry() { - return certificateRetry; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java deleted file mode 100644 index c72c2f26a1c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.collect.ImmutableList; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.federation.FederationSearcher; -import com.yahoo.search.query.Properties; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.statistics.Value; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.*; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Superclass of searchers which talks to HTTP backends. Implement a subclass to talk to a backend - * over HTTP which is not supported by the platform out of the box. - * <p> - * Implementations must override one of the <code>unmarshal</code> methods to unmarshal the response. - * </p> - * - * @author Arne Bergene Fossaa - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPProviderSearcher extends HTTPSearcher { - - private final Counter emptyResults; - private final Value hitsPerQuery; - private final Value responseLatency; - private final Counter readTimeouts; - - private final static List<String> excludedSourceProperties = ImmutableList.of("offset", "hits", "provider"); - - protected final static Logger log = Logger.getLogger(HTTPProviderSearcher.class.getName()); - - /** The name of the cache used (which is just getid().stringValue(), or null if no cache is used */ - protected String cacheName=null; - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections,String path, Statistics statistics) { - this(id,connections,new HTTPParameters(path), statistics); - } - - /** Creates a http provider searcher using id.getName as provider name */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, String path, - Statistics statistics, CertificateStore certificateStore) { - this(id, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(id, connections, parameters, statistics, new ThrowingCertificateStore()); - } - - /** - * Creates a provider searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - String suffix = "_" + getId().getName().replace('.', '_'); - hitsPerQuery = new Value("hits_per_query" + suffix, statistics, - new Value.Parameters().setLogRaw(false).setNameExtension(false).setLogMean(true)); - responseLatency = new Value(LOG_LATENCY_START + suffix, statistics, - new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false)); - emptyResults = new Counter("empty_results" + suffix, statistics, false); - readTimeouts = new Counter(LOG_READ_TIMEOUT_PREFIX + suffix, statistics, false); - } - - /** @deprecated this method does nothing */ - @Deprecated // OK - protected void configureCache(final QrBinaryCacheConfig cacheConfig,final QrBinaryCacheRegionConfig regionConfig) { - } - - /** - * Unmarshal the stream by converting it to hits and adding the hits to the given result. - * A convenience hook called by the default <code>unmarshal(entity,result).</code> - * Override this in subclasses which does not override <code>unmarshal(entity,result).</code> - * <p> - * This default implementation throws an exception. - * - * @param stream the stream of data returned - * @param contentLength the length of the content in bytes if known, or a negative number if unknown - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { - throw new UnsupportedOperationException("Unmarshal must be implemented by " + this); - } - - /** - * Unmarshal the result from an http entity. This default implementation calls - * <code>unmarshal(entity.getContent(), entity.getContentLength(), result)</code> - * (and does some detailed query tracing). - * - * @param entity the entity containing the data to unmarshal - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(HttpEntity entity,Result result) throws IOException { - Query query=result.getQuery(); - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query.trace("Unmarshaling result.", false, 6); - unmarshal(entity.getContent(), len, result); - - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - - } - - protected void addNonExcludedSourceProperties(Query query, Map<String, String> queryMap) { - Properties sourceProperties = FederationSearcher.getSourceProperties(query); - if (sourceProperties != null) { - for(Map.Entry<String, Object> entry : sourceProperties.listProperties("").entrySet()) { - if (!excludedSourceProperties.contains(entry.getKey())) { - queryMap.put(entry.getKey(), entry.getValue().toString()); - } - } - } - } - - /** - * Hook called at the moment the result is returned from this searcher. This default implementation - * does <code>return result</code>. - * - * @param result the result which is to be returned - * @param requestMeta the request information hit, or null if none was created (e.g if this was a cache lookup) - * @param e the exception caused during execution of this query, or null if none - * @return the result which is returned upwards - */ - protected Result inspectAndReturnFinalResult(Result result, Hit requestMeta, Exception e) { - return result; - } - - private Result statisticsBeforeInspection(Result result, Hit requestMeta, Exception e) { - int hitCount = result.getConcreteHitCount(); - if (hitCount == 0) { - emptyResults.increment(); - } - hitsPerQuery.put((double) hitCount); - - if (requestMeta != null) { - requestMeta.setField(LOG_HITCOUNT, Integer.valueOf(hitCount)); - } - - return inspectAndReturnFinalResult(result, requestMeta, e); - } - - - @Override - protected void logResponseLatency(long latency) { - responseLatency.put((double) latency); - } - - @Override - public Result search(Query query, Execution execution,Connection connection) { - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - Result result = new Result(query); - result.hits().add(requestMeta); - query.trace("Created request information hit", false, 9); - - try { - URI uri = getURI(query, requestMeta, connection); - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - long requestStartTime = System.currentTimeMillis(); - - HttpEntity entity = getEntity(uri, requestMeta, query); - - // Why should consumeEntity call inspectAndReturnFinalResult itself? - // Seems confusing to me. - return entity == null - ? statisticsBeforeInspection(result, requestMeta, null) - : consumeEntity(entity, query, result, requestMeta, requestStartTime); - - } catch (MalformedURLException|URISyntaxException e) { - result.hits().addError(createMalformedUrlError(query,e)); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (TimeoutException e) { - result.hits().addError(ErrorMessage.createTimeout("No time left for HTTP traffic in " - + this - + " for " + query + ": " + e.getMessage())); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this - + " for " + query + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, requestMeta, e); - } - } - - private Result consumeEntity(HttpEntity entity, Query query, Result result, Hit logHit, long requestStartTime) { - - try { - // remove some time from timeout to allow for close calls with return result - unmarshal(new TimedHttpEntity(entity, query.getStartTime(), Math.max(1, query.getTimeout() - 10)), result); - logHit.setField(LOG_LATENCY_FINISH, System.currentTimeMillis() - requestStartTime); - return statisticsBeforeInspection(result, logHit, null); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, logHit, e); - } catch (TimeoutException e) { - readTimeouts.increment(); - result.hits().addError(ErrorMessage - .createTimeout("Timed out while reading/unmarshaling from backend in " - + this + " for " + query - + ": " + e.getMessage())); - return statisticsBeforeInspection(result, logHit, e); - } finally { // TODO: The scope of this finally must be enlarged to release the connection also on errors - cleanupHttpEntity(entity); - } - } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns the query.properties() prefixed by - * "source.[sourceName]" or "property.[propertyName]" - * (by calling {@link #addNonExcludedSourceProperties}). - */ - @Override - public Map<String,String> getQueryMap(Query query) { - Map<String,String> queryMap = super.getQueryMap(query); - addNonExcludedSourceProperties(query, queryMap); - return queryMap; - } - - /** - * @deprecated the cache key is ignored as there is no built-in caching support - */ - @Deprecated // OK - public abstract Map<String, String> getCacheKey(Query q); - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java deleted file mode 100644 index edf347bd84e..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java +++ /dev/null @@ -1,961 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.inject.Inject; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.cluster.ClusterSearcher; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; - -import org.apache.http.*; -import org.apache.http.client.HttpClient; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnRoutePNames; -import org.apache.http.conn.routing.HttpRoutePlanner; -import org.apache.http.conn.scheme.PlainSocketFactory; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.scheme.SchemeRegistry; -import org.apache.http.conn.ssl.SSLSocketFactory; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.DefaultHttpRoutePlanner; -import org.apache.http.impl.conn.SingleClientConnManager; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.HttpParams; -import org.apache.http.params.HttpProtocolParams; -import org.apache.http.protocol.BasicHttpContext; -import org.apache.http.protocol.ExecutionContext; -import org.apache.http.protocol.HttpContext; -import org.apache.http.protocol.HttpRequestExecutor; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.SSLHandshakeException; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.UnsupportedEncodingException; -import java.net.*; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Generic superclass of searchers making connections to some HTTP service. This - * supports clustered connections - a list of alternative servers may be given, - * requests will be hashed across these and failed over in case some are down. - * <p> - * This simply provides some utility methods for working with http connections - * and implements ping against the service. - * - * <p>This searcher contains code from the Apache httpcomponents client library, - * licensed to the Apache Software Foundation under the Apache License, Version - * 2.0. Please refer to http://www.apache.org/licenses/LICENSE-2.0 for details. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, as this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_HITCOUNT - * <dd> - * Number of concrete hits in the result returned by this provider. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - * - * @author Arne Bergene Fossaa - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPSearcher extends ClusterSearcher<Connection> { - - protected static final String YCA_HTTP_HEADER = "Yahoo-App-Auth"; - - private static final Charset iso8859Charset = Charset.forName("ISO-8859-1"); - - // Logging field name constants - public static final String LOG_PATH = "path"; - public static final String LOG_PORT = "port"; - public static final String LOG_HOST = "host"; - public static final String LOG_IP_ADDRESS = "ip_address"; - public static final String IP_ADDRESS_UNKNOWN = "unknown"; - - public static final String LOG_SCHEME = "scheme"; - public static final String LOG_URI = "uri"; - public static final String LOG_PROXY_PORT = "proxy_port"; - public static final String LOG_PROXY_HOST = "proxy_host"; - public static final String LOG_PROXY_TYPE = "proxy_type"; - public static final String LOG_STATUS = "status"; - public static final String LOG_LATENCY_FINISH = "latency_finish"; - public static final String LOG_LATENCY_START = "latency_start"; - public static final String LOG_LATENCY_CONNECT = "latency_connect"; - public static final String LOG_QUERY_PARAM_PREFIX = "query_param_"; - public static final String LOG_HEADER_PREFIX = "header_"; - public static final String LOG_RESPONSE_HEADER_PREFIX = "response_header_"; - public static final String LOG_HITCOUNT = "hit_count"; - public static final String LOG_CONNECT_TIMEOUT_PREFIX = "connect_timeout_"; - public static final String LOG_READ_TIMEOUT_PREFIX = "read_timeout_"; - - protected final Logger log = Logger.getLogger(HTTPSearcher.class.getName()); - - /** The HTTP parameters to use. Assigned in the constructor */ - private HTTPParameters httpParameters; - - private final Counter connectTimeouts; - - /** Whether to use certificates */ - protected boolean useCertificate = false; - - private final CertificateStore certificateStore; - - /** The (optional) certificate application ID. */ - private String certificateApplicationId = null; - - /** The (optional) certificate server proxy */ - protected HttpHost certificateProxy = null; - - /** Certificate cache TTL in ms */ - private long certificateTtl = 0L; - - /** Certificate server retry rate in the cache if no cert is found, in ms */ - private long certificateRetry = 0L; - - /** Set at construction if this is using persistent connections */ - private ClientConnectionManager sharedConnectionManager = null; - - /** Set at construction if using non-persistent connections */ - private ThreadLocal<SingleClientConnManager> singleClientConnManagerThreadLocal = null; - - private static final SchemeRegistry schemeRegistry = new SchemeRegistry(); - - static { - schemeRegistry.register(new Scheme("http", PlainSocketFactory - .getSocketFactory(), 80)); - schemeRegistry.register(new Scheme("https", SSLSocketFactory - .getSocketFactory(), 443)); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics) { - this(componentId, connections, new HTTPParameters(path), statistics, new ThrowingCertificateStore()); - } - - /** Creates a http searcher with default connection and read timeouts (currently 2 and 5s respectively) */ - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics, - CertificateStore certificateStore) { - this(componentId, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(componentId, connections, parameters, statistics, new ThrowingCertificateStore()); - } - /** - * Creates a http searcher - * - * @param componentId the id of this instance - * @param connections the connections to establish to the backend nodes - * @param parameters the http parameters to use. This object will be frozen if it isn't already - */ - @Inject - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(componentId,connections,false); - String suffix = "_" + getId().getName().replace('.', '_'); - - connectTimeouts = new Counter(LOG_CONNECT_TIMEOUT_PREFIX + suffix, statistics, false); - - parameters.freeze(); - this.httpParameters = parameters; - this.certificateStore = certificateStore; - - if (parameters.getPersistentConnections()) { - HttpParams params=parameters.toHttpParams(); - HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); - ConnManagerParams.setTimeout(params, 10); - sharedConnectionManager = new ThreadSafeClientConnManager(params, schemeRegistry); - Thread connectionPurgerThread = new Thread(() -> { - //this is the default value in yahoo jvm installations - long DNSTTLSec = 120; - while (true) { - try { - Thread.sleep(DNSTTLSec * 1000); - if (sharedConnectionManager == null) - continue; - - sharedConnectionManager.closeExpiredConnections(); - DNSTTLSec = Long.valueOf(java.security.Security - .getProperty("networkaddress.cache.ttl")); - //No DNS TTL, no need to close idle connections - if (DNSTTLSec <= 0) { - DNSTTLSec = 120; - continue; - } - sharedConnectionManager.closeIdleConnections(2 * DNSTTLSec, TimeUnit.SECONDS); - } catch (InterruptedException e) { - return; - } catch (NumberFormatException e) { - continue; - } - } - }); - connectionPurgerThread.setDaemon(true); - connectionPurgerThread.start(); - - } - else { - singleClientConnManagerThreadLocal =new ThreadLocal<>(); - } - - initializeCertificate(httpParameters, certificateStore); - } - - /** - * Initialize certificate store and proxy if they have been set to non-null, - * non-empty values. It will wrap thrown exceptions from the certificate store into - * RuntimeException and propagate them. - */ - private void initializeCertificate(HTTPParameters parameters, CertificateStore certificateStore) { - String applicationId = parameters.getYcaApplicationId(); - String proxy = parameters.getYcaProxy(); - int port = parameters.getYcaPort(); - long ttl = parameters.getYcaTtl(); - long retry = parameters.getYcaRetry(); - - if (applicationId != null && !applicationId.trim().isEmpty()) { - initializeCertificate(applicationId, ttl, retry, certificateStore); - } - - if (parameters.getYcaUseProxy()) { - initializeProxy(proxy, port); - } - } - - /** Returns the HTTP parameters used in this. This is always frozen */ - public HTTPParameters getParameters() { return httpParameters; } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns an empty LinkedHashMap. - */ - public Map<String,String> getQueryMap(Query query) { - return new LinkedHashMap<>(); - } - - /** - * Initialize the certificate. - * This will warn but not throw if certificates could not be loaded, as the certificates - * are external state which can fail independently. - */ - private void initializeCertificate(String applicationId, long ttl, long retry, CertificateStore certificateStore) { - try { - // get the certificate, i.e. init the cache and check integrity - String certificate = certificateStore.getCertificate(applicationId, ttl, retry); - if (certificate == null) { - getLogger().log(LogLevel.WARNING, "No certificate found for application '" + applicationId + "'"); - return; - } - - this.useCertificate = true; - this.certificateApplicationId = applicationId; - this.certificateTtl = ttl; - this.certificateRetry = retry; - getLogger().log(LogLevel.CONFIG, "Got certificate: " + certificate); - } - catch (Exception e) { - getLogger().log(LogLevel.WARNING,"Exception while initializing certificate for application '" + - applicationId + "' in " + this, e); - } - } - - /** - * Initialize the certificate proxy setting. - */ - private void initializeProxy(String host, int port) { - certificateProxy = new HttpHost(host, port); - getLogger().log(LogLevel.CONFIG, "Proxy is configured; will use proxy: " + certificateProxy); - } - - /** - * Same a {@code getURI(query, offset, hits, null)}. - * @see #getURI(Query, Hit, Connection) - */ - protected URI getURI(Query query,Connection connection) throws MalformedURLException, URISyntaxException { - Hit requestMeta; - try { - requestMeta = (Hit) query.properties().get(HTTPClientSearcher.REQUEST_META_CARRIER); - } catch (ClassCastException e) { - requestMeta = null; - } - return getURI(query, requestMeta, connection); - } - - /** - * Creates the URI for a query. - * Populates the {@code requestMeta} meta hit with the created URI HTTP properties. - * - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - */ - protected URI getURI(Query query, Hit requestMeta, Connection connection) - throws MalformedURLException, URISyntaxException { - StringBuilder parameters = new StringBuilder(); - - Map<String, String> queries = getQueryMap(query); - if (queries.size() > 0) { - Iterator<Map.Entry<String, String>> mapIterator = queries.entrySet().iterator(); - parameters.append("?"); - try { - Map.Entry<String, String> entry; - while (mapIterator.hasNext()) { - entry = mapIterator.next(); - - if (requestMeta != null) - requestMeta.setField(LOG_QUERY_PARAM_PREFIX - + entry.getKey(), entry.getValue()); - - parameters.append(entry.getKey() + "=" + URLEncoder.encode(entry.getValue(), - httpParameters.getInputEncoding())); - if (mapIterator.hasNext()) { - parameters.append("&"); - } - } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Unknown input encoding set in " + this, e); - } - } - - URI uri = new URL(httpParameters.getSchema(), connection.getHost(), - connection.getPort(), getPath() + parameters.toString()).toURI(); - if (requestMeta != null) { - requestMeta.setField(LOG_URI, uri.toString()); - requestMeta.setField(LOG_SCHEME, uri.getScheme()); - requestMeta.setField(LOG_HOST, uri.getHost()); - requestMeta.setField(LOG_PORT, uri.getPort()); - requestMeta.setField(LOG_PATH, uri.getPath()); - } - return uri; - } - - /** - * Called by getURI() to get the path of the URI for the external service. - * The default implementation returns httpParameters.getPath(); subclasses - * which only wants to override the path from httpParameters may use this - * method instead of overriding all of getURI(). - * - * @return the path to use for getURI - */ - protected String getPath() { - return httpParameters.getPath(); - } - - /** - * The URI that is used to check if the provider is up or down. This will again be used in the - * checkPing method by checking that we get a response that has a good status code (below 300). If better - * validation than just status code checking is needed, override the checkPing method. - */ - protected URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException { - return new URL(httpParameters.getSchema(),connection.getHost(),connection.getPort(),getPingPath()).toURI(); - } - - /** - * Called by getPingURI() to get the path of the URI for pinging the - * external service. The default implementation returns - * httpParameters.getPath(); subclasses which only wants to override the - * path from httpParameters may use this method instead of overriding all of - * getPingURI(). - * - * @return the path to use for getPingURI - */ - protected String getPingPath() { - return httpParameters.getPath(); - } - - /** - * Checks if the response is valid. - * @param response The response from the ping request - * @param pong The pong result to return back to the calling method. This method - * will add an error to the pong result (using addError) if the status of the HTTP response is 300 or above. - */ - protected void checkPing(HttpResponse response, Pong pong) { - if (response.getStatusLine().getStatusCode() >= 300) { - pong.addError(com.yahoo.search.result.ErrorMessage.createBackendCommunicationError( - "Got error " + response.getStatusLine().getStatusCode() - + " when contacting backend") - ); - } - } - - /** - * Pinging in HTTPBackend is done by creating a PING uri from http://host:port/path. - * If this returns a status that is below 300, the ping is considered good. - * - * If another uri is needed for pinging, reimplement getPingURI. - * - * Override either this method to change how ping - */ - @Override - public Pong ping(Ping ping, Connection connection) { - URI uri = null; - Pong pong = new Pong(); - HttpResponse response = null; - - if (httpParameters.getPingOption() == PingOption.DISABLE) - return pong; - - try { - uri = getPingURI(connection); - if (uri == null) - pong.addError(ErrorMessage.createIllegalQuery("Ping uri is null")); - if (uri.getHost()==null) { - pong.addError(ErrorMessage.createIllegalQuery("Ping uri has no host")); - uri=null; - } - } catch (MalformedURLException | URISyntaxException e) { - pong.addError(ErrorMessage.createIllegalQuery("Malformed ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + - " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } - - if (uri == null) return pong; - pong.setPingInfo("using uri '" + uri + "'"); - - try { - response = getPingResponse(uri, ping); - checkPing(response, pong); - } catch (IOException e) { - // We do not have a valid ping - pong.addError(ErrorMessage.createBackendCommunicationError( - "Exception thrown when pinging with url '" + uri + "': " + Exceptions.toMessageString(e))); - } catch (TimeoutException e) { - pong.addError(ErrorMessage.createTimeout("Timeout for ping " + uri + " in " + this + ": " + e.getMessage())); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } finally { - if (response != null) { - cleanupHttpEntity(response.getEntity()); - } - } - - return pong; - } - - private HttpResponse getPingResponse(URI uri, Ping ping) throws IOException { - long timeLeft = ping.getTimeout(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - - Map<String, String> requestHeaders = null; - if (httpParameters.getPingOption() == PingOption.YCA) - requestHeaders = generateYCAHeaders(); - - return getResponse(uri, null, requestHeaders, null, connectionTimeout, readTimeout); - } - - /** - * Same a {@code getEntity(uri, null)}. - * @param uri resource to fetch - * @param query the originating query - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Query query) throws IOException{ - return getEntity(uri, null, query); - } - - - /** - * Gets the HTTP entity that holds the response contents. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query - * @return the http entity, or null if none - * @throws java.io.IOException Whenever HTTP status code is in the 300 or higher range. - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { - if (query.getTimeLeft() <= 0) { - throw new TimeoutException("No time left for querying external backend."); - } - HttpResponse response = getResponse(uri, requestMeta, query); - StatusLine statusLine = response.getStatusLine(); - - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_STATUS, statusLine.getStatusCode()); - for (HeaderIterator headers = response.headerIterator(); headers.hasNext(); ) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_RESPONSE_HEADER_PREFIX + h.getName(), h.getValue()); - } - } - - if (statusLine.getStatusCode() >= 300) { - HttpEntity entity = response.getEntity(); - String message = createServerReporterErrorMessage(statusLine, entity); - cleanupHttpEntity(response.getEntity()); - throw new IOException(message); - } - - return response.getEntity(); - } - - private String createServerReporterErrorMessage(StatusLine statusLine, HttpEntity entity) { - String message = "Error when trying to connect to HTTP backend: " - + statusLine.getStatusCode() + " : " + statusLine.getReasonPhrase(); - - try { - if (entity != null) { - message += "(Message = " + EntityUtils.toString(entity) + ")"; - } - } catch (Exception e) { - log.log(LogLevel.WARNING, "Could not get message.", e); - } - - return message; - } - - /** - * Creates a meta hit dedicated to holding logging information. This hit has - * the 'logging:[searcher's ID]' type. - */ - protected Hit createRequestMeta() { - Hit requestMeta = new Hit("logging:" + getId().toString()); - requestMeta.setMeta(true); - requestMeta.types().add("logging"); - return requestMeta; - } - - protected void cleanupHttpEntity(HttpEntity entity) { - if (entity == null) return; - - try { - entity.consumeContent(); - } catch (IOException e) { - // It is ok if do not consume it, the resource will be freed after - // timeout. - // But log it just in case. - log.log(LogLevel.getVespaLogLevel(LogLevel.DEBUG), - "Not able to consume after processing: " + Exceptions.toMessageString(e)); - } - } - - /** - * Same as {@code getResponse(uri, null)}. - */ - protected HttpResponse getResponse(URI uri, Query query) throws IOException{ - return getResponse(uri, null, query); - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query, used to calculate timeouts - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, Query query) throws IOException { - long timeLeft = query.getTimeLeft(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - connectionTimeout = connectionTimeout <= 0 ? 1 : connectionTimeout; - readTimeout = readTimeout <= 0 ? 1 : readTimeout; - HttpEntity reqEntity = getRequestEntity(query, requestMeta); - Map<String, String> reqHeaders = getRequestHeaders(query, requestMeta); - if ((reqEntity == null) && (reqHeaders == null)) { - return getResponse(uri, requestMeta, connectionTimeout, readTimeout); - } else { - return getResponse(uri, reqEntity, reqHeaders, requestMeta, connectionTimeout, readTimeout); - } - } - - /** - * Returns the set of headers to be passed in the http request to provider backend. The default - * implementation returns null, unless certificates are in use. If certificates are used, it will return a map - * only containing the needed certificate headers. - */ - protected Map<String, String> getRequestHeaders(Query query, Hit requestMeta) { - if (useCertificate) { - return generateYCAHeaders(); - } - return null; - } - - /** - * Returns the HTTP request entity to use when making the request for this query. - * This default implementation returns null. - * - * <p> Do return a repeatable entity if HTTP retry is active. - * - * @return the http request entity to use, or null to use the default entity - */ - protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { - return null; - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, int connectionTimeout, int readTimeout) - throws IOException { - return getResponse(uri, null, null, requestMeta, connectionTimeout, readTimeout); - } - - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, HttpEntity reqEntity, - Map<String, String> reqHeaders, Hit requestMeta, - int connectionTimeout, int readTimeout) throws IOException { - - HttpParams httpParams = httpParameters.toHttpParams(connectionTimeout, readTimeout); - HttpClient httpClient = createClient(httpParams); - long start = 0L; - HttpUriRequest request; - if (httpParameters.getEnableProxy() && "http".equals(httpParameters.getProxyType())) { - HttpHost proxy = new HttpHost(httpParameters.getProxyHost(), - httpParameters.getProxyPort(), httpParameters.getProxyType()); - httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_PROXY_TYPE, httpParameters.getProxyType()); - requestMeta.setField(LOG_PROXY_HOST, httpParameters.getProxyHost()); - requestMeta.setField(LOG_PROXY_PORT, httpParameters.getProxyPort()); - } - } - if (reqEntity == null) { - request = createRequest(httpParameters.getMethod(), uri); - } else { - request = createRequest(httpParameters.getMethod(), uri, reqEntity); - } - - if (reqHeaders != null) { - for (Entry<String, String> entry : reqHeaders.entrySet()) { - if (entry.getValue() == null || isAscii(entry.getValue())) { - request.addHeader(entry.getKey(), entry.getValue()); - } else { - byte[] asBytes = Utf8.toBytes(entry.getValue()); - String asLyingString = new String(asBytes, 0, asBytes.length, iso8859Charset); - request.addHeader(entry.getKey(), asLyingString); - } - } - } - - // Logging - if (requestMeta != null) { - for (HeaderIterator headers = request.headerIterator(); headers.hasNext();) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_HEADER_PREFIX + h.getName(), h.getValue()); - } - start = System.currentTimeMillis(); - } - - HttpResponse response; - - try { - HttpContext context = new BasicHttpContext(); - response = httpClient.execute(request, context); - - if (requestMeta != null) { - requestMeta.setField(LOG_IP_ADDRESS, getIpAddress(context)); - } - } catch (ConnectTimeoutException e) { - connectTimeouts.increment(); - throw e; - } - - // Logging - long latencyStart = System.currentTimeMillis() - start; - if (requestMeta != null) { - requestMeta.setField(LOG_LATENCY_START, latencyStart); - } - logResponseLatency(latencyStart); - return response; - } - - private String getIpAddress(HttpContext context) { - HttpConnection connection = (HttpConnection) context.getAttribute(ExecutionContext.HTTP_CONNECTION); - if (connection instanceof HttpInetConnection) { - InetAddress address = ((HttpInetConnection) connection).getRemoteAddress(); - String hostAddress = address.getHostAddress(); - return hostAddress == null ? - IP_ADDRESS_UNKNOWN: - hostAddress; - } else { - getLogger().log(LogLevel.DEBUG, "Unexpected connection type: " + connection.getClass().getName()); - return IP_ADDRESS_UNKNOWN; - } - } - - private boolean isAscii(String value) { - char[] scanBuffer = new char[value.length()]; - value.getChars(0, value.length(), scanBuffer, 0); - for (char c: scanBuffer) - if (c > 127) return false; - return true; - } - - protected void logResponseLatency(long latency) { } - - /** - * Creates a http client for one request. Override to customize the client - * to use, e.g for testing. This default implementation will add a certificate store - * proxy to params if is necessary, and then do - * <code>return new SearcherHttpClient(getConnectionManager(params), params);</code> - */ - protected HttpClient createClient(HttpParams params) { - if (certificateProxy != null) { - params.setParameter(ConnRoutePNames.DEFAULT_PROXY, certificateProxy); - } - return new SearcherHttpClient(getConnectionManager(params), params); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri) { - return createRequest(method, uri, null); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri, HttpEntity entity) { - return new SearcherHttpRequest(method,uri); - } - - /** Get a connection manager which may be used safely from this thread */ - protected ClientConnectionManager getConnectionManager(HttpParams params) { - if (sharedConnectionManager != null) {// We are using shared connections - return sharedConnectionManager; - } else { - SingleClientConnManager singleClientConnManager = singleClientConnManagerThreadLocal.get(); - if (singleClientConnManager == null) { - singleClientConnManager = new SingleClientConnManager(params, schemeRegistry); - singleClientConnManagerThreadLocal.set(singleClientConnManager); - } - return singleClientConnManager; - } - } - - /** Utility method for creating error messages when a url is incorrect */ - protected ErrorMessage createMalformedUrlError(Query query,Exception e) { - return ErrorMessage.createErrorInPluginSearcher("Malformed url in " + this + " for " + query + - ": " + Exceptions.toMessageString(e)); - } - - private Map<String, String> generateYCAHeaders() { - Map<String, String> headers = new HashMap<>(); - String certificate = certificateStore.getCertificate(certificateApplicationId, certificateTtl, certificateRetry); - headers.put(YCA_HTTP_HEADER, certificate); - return headers; - } - - protected static class SearcherHttpClient extends DefaultHttpClient { - - private final int retries; - - public SearcherHttpClient(final ClientConnectionManager conman, final HttpParams params) { - super(conman, params); - retries = params.getIntParameter(HTTPParameters.RETRIES, 1); - addRequestInterceptor((request, context) -> { - if (!request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip"); - } - }); - addResponseInterceptor((response, context) -> { - HttpEntity entity = response.getEntity(); - if (entity == null) return; - Header ceheader = entity.getContentEncoding(); - if (ceheader == null) return; - for (HeaderElement codec : ceheader.getElements()) { - if (codec.getName().equalsIgnoreCase("gzip")) { - response.setEntity(new GzipDecompressingEntity(response.getEntity())); - return; - } - } - }); - } - - @Override - protected HttpRequestExecutor createRequestExecutor() { - return new HttpRequestExecutor(); - } - - @Override - protected HttpRoutePlanner createHttpRoutePlanner() { - return new DefaultHttpRoutePlanner(getConnectionManager().getSchemeRegistry()); - } - - @Override - protected HttpRequestRetryHandler createHttpRequestRetryHandler() { - return new SearcherHttpRequestRetryHandler(retries); - } - } - - /** A retry handler which avoids retrying forever on errors misclassified as transient */ - private static class SearcherHttpRequestRetryHandler implements HttpRequestRetryHandler { - private final int retries; - - public SearcherHttpRequestRetryHandler(int retries) { - this.retries = retries; - } - - @Override - public boolean retryRequest(IOException e, int executionCount, HttpContext httpContext) { - if (e == null) { - throw new IllegalArgumentException("Exception parameter may not be null"); - } - if (executionCount > retries) { - return false; - } - if (e instanceof NoHttpResponseException) { - // Retry if the server dropped connection on us - return true; - } - if (e instanceof InterruptedIOException) { - // Timeout from federation layer - return false; - } - if (e instanceof UnknownHostException) { - // Unknown host - return false; - } - if (e instanceof SSLHandshakeException) { - // SSL handshake exception - return false; - } - return true; - } - - - } - - private static class SearcherHttpRequest extends HttpRequestBase { - String method; - - public SearcherHttpRequest(String method, final URI uri) { - super(); - this.method = method; - setURI(uri); - } - - @Override - public String getMethod() { - return method; - } - } - - /** - * Only for testing. - */ - public void shutdownConnectionManagers() { - ClientConnectionManager manager; - if (sharedConnectionManager != null) { - manager = sharedConnectionManager; - } else { - manager = singleClientConnManagerThreadLocal.get(); - } - if (manager != null) { - manager.shutdown(); - } - } - - protected static final class ThrowingCertificateStore implements CertificateStore { - - @Override - public String getCertificate(String key, long ttl, long retry) { - throw new UnsupportedOperationException("A certificate store is not available"); - } - - } - -} - diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java deleted file mode 100644 index 03ffe2b8a9c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import org.apache.http.Header; -import org.apache.http.HttpEntity; - -/** - * Wrapper for adding timeout to an HttpEntity instance. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedHttpEntity implements HttpEntity { - /** - * The wrapped entity. Never null. - */ - private final HttpEntity entity; - private final long startTime; - private final long timeout; - - public TimedHttpEntity(HttpEntity entity, long startTime, long timeout) { - if (entity == null) { - throw new IllegalArgumentException("TimedHttpEntity cannot be instantiated with null HttpEntity."); - } - this.entity = entity; - this.startTime = startTime; - this.timeout = timeout; - } - - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - InputStream content = entity.getContent(); - if (content == null) { - return null; - } else { - return new TimedStream(content, startTime, timeout); - } - } - - - // START OF PURE FORWARDING METHODS - @Override - public void consumeContent() throws IOException { - entity.consumeContent(); - } - - - @Override - public Header getContentEncoding() { - return entity.getContentEncoding(); - } - - @Override - public long getContentLength() { - return entity.getContentLength(); - } - - @Override - public Header getContentType() { - return entity.getContentType(); - } - - @Override - public boolean isChunked() { - return entity.isChunked(); - } - - @Override - public boolean isRepeatable() { - return entity.isRepeatable(); - } - - @Override - public boolean isStreaming() { - return entity.isStreaming(); - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - entity.writeTo(outstream); - } - // END OF PURE FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java deleted file mode 100644 index 77a42ee0a34..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; - -/** - * A stream which throws a TimeoutException if query timeout has been reached. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedStream extends InputStream { - - /** - * A time barrier value, the point in time from which on read operations will cause an exception. - */ - private final long limit; - - /** - * A wrapped InputStream instance. - */ - private final InputStream content; - - /** - * Wrap an InputStream to make read operations potentially fire off - * TimeoutException. - * - * <p>Typical use would be<br> - * <code>new TimedStream(httpEntity.getContent(), query.getStartTime(), query.getTimeout())</code> - * - * @param content - * the InputStream to wrap - * @param startTime - * start time of query - * @param timeout - * how long the query is allowed to run - */ - public TimedStream(InputStream content, long startTime, long timeout) { - if (content == null) { - throw new IllegalArgumentException("Cannot instantiate TimedStream with null InputStream"); - } - this.content = content; - // The reasion for doing it in here instead of outside the constructor - // is this makes the usage of the class more intuitive IMHO - this.limit = startTime + timeout; - } - - private void checkTime(String message) { - if (System.currentTimeMillis() >= limit) { - throw new TimeoutException(message); - } - } - - // START FORWARDING METHODS: - // All methods below are forwarding methods to the contained stream, where - // some do a timeout check. - @Override - public int read() throws IOException { - int data = content.read(); - checkTime("Timed out during read()."); - return data; - } - - @Override - public int available() throws IOException { - return content.available(); - } - - @Override - public void close() throws IOException { - content.close(); - } - - @Override - public synchronized void mark(int readlimit) { - content.mark(readlimit); - } - - @Override - public boolean markSupported() { - return content.markSupported(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int length = content.read(b, off, len); - checkTime("Timed out during read(byte[], int, int)"); - return length; - } - - @Override - public int read(byte[] b) throws IOException { - int length = content.read(b); - checkTime("Timed out during read(byte[])"); - return length; - } - - @Override - public synchronized void reset() throws IOException { - content.reset(); - } - - @Override - public long skip(long n) throws IOException { - long skipped = content.skip(n); - checkTime("Timed out during skip(long)"); - return skipped; - } - // END FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java deleted file mode 100644 index 18dd59ec37b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Timeout marker for slow HTTP connections. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimeoutException extends RuntimeException { - - /** - * Auto-generated version ID. - */ - private static final long serialVersionUID = 7084147598258586559L; - - public TimeoutException(String message) { - super(message); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java deleted file mode 100644 index af334b35221..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.search.federation.http; - -import com.yahoo.osgi.annotation.ExportPackage; |