diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-11-05 13:33:19 +0100 |
---|---|---|
committer | gjoranv <gv@oath.com> | 2019-01-21 15:09:25 +0100 |
commit | 8505bd016ec028c4dcdeb04ec4baa07a435e166b (patch) | |
tree | e63e493940d11732214b240269f5238cfe7e6903 /container-search | |
parent | 6b01125fad374672738a0e68edf7eef5f5463ec9 (diff) |
Remove HTTP searchers
Diffstat (limited to 'container-search')
37 files changed, 0 insertions, 5906 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java deleted file mode 100644 index 1607d108722..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.Collections; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - - -/** - * Superclass for http client searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPClientSearcher extends HTTPClientSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPClientSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id, ConfiguredSearcherHelper.toConnectionList(providerConfig), new HTTPParameters(providerConfig), manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPClientSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Forwards to the next in chain fill(result,summaryName) */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - execution.fill(result,summaryName); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java deleted file mode 100644 index 9f99a790df0..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import java.util.Collections; - - -/** - * Superclass for http provider searchers which depends on config. All this is doing is translating - * the provider and cache configurations to parameters which are passed upwards. - * - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class ConfiguredHTTPProviderSearcher extends HTTPProviderSearcher { - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - } - - /** Create this from a configuraton */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create this from a configuraton with a configured cache */ - public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, - final QrBinaryCacheConfig cacheConfig, - final QrBinaryCacheRegionConfig regionConfig, HTTPParameters parameters, Statistics manager) { - super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager); - configureCache(cacheConfig,regionConfig); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,String path, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager); - } - - /** Create an instance from direct parameters having a single connection. Useful for testing */ - public ConfiguredHTTPProviderSearcher(String idString,String host,int port,HTTPParameters parameters, Statistics manager) { - super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),parameters, manager); - } - - /** - * Override this to provider multi-phase result filling towards a backend. - * This default implementation does nothing. - */ - @Override - public void fill(Result result,String summaryName, Execution execution,Connection connection) { - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java deleted file mode 100644 index 0fa235709f9..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * Some static helper classes for configured*Searcher classes - * - * @author bratseth - */ -// TODO: Remove on Vespa 7 -class ConfiguredSearcherHelper { - - /** No instantiation */ - private ConfiguredSearcherHelper() { } - - public static List<Connection> toConnectionList(ProviderConfig providerConfig) { - List<Connection> connections=new ArrayList<>(); - for(ProviderConfig.Node node : providerConfig.node()) { - connections.add(new Connection(node.host(), node.port())); - } - return connections; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java b/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java deleted file mode 100644 index 4f624a12547..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Represents a connection to a particular node (host/port). - * Right now this is just a container of connection parameters, but might be extended to - * contain an open connection later. - * The host and port state is immutable. - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class Connection { - - private String host; - private int port; - - public Connection(String host,int port) { - this.host=host; - this.port=port; - } - - public String getHost() { return host; } - - public int getPort() { return port; } - - public String toString() { - return "http connection '" + host + ":" + port + "'"; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java deleted file mode 100644 index 0407ddeca1b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import org.apache.http.HttpEntity; -import org.apache.http.entity.HttpEntityWrapper; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.zip.GZIPInputStream; - -/** - * Used by HTTPSearcher when talking to services returning compressed content. - * - * @author Mainak Mandal - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class GzipDecompressingEntity extends HttpEntityWrapper { - - private static class Resources { - - byte [] buffer; - int total; - - Resources() { - total = 0; - buffer = new byte[65536]; - } - void drain(InputStream zipStream) throws IOException { - int numRead = zipStream.read(buffer, total, buffer.length); - while (numRead != -1) { - total += numRead; - if ((total + 65536) > buffer.length) { - buffer = Arrays.copyOf(buffer, buffer.length + numRead); - } - numRead = zipStream.read(buffer, total, buffer.length - total); - } - } - - } - - private final Resources resources = new Resources(); - - public GzipDecompressingEntity(final HttpEntity entity) throws IllegalStateException, IOException { - super(entity); - GZIPInputStream gz = new GZIPInputStream(entity.getContent()); - InputStream zipStream = new BufferedInputStream(gz); - try { - resources.drain(zipStream); - } catch (IOException e) { - throw e; - } finally { - zipStream.close(); - } - } - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - - final ByteBuffer buff = ByteBuffer.wrap(resources.buffer, 0, resources.total); - return new InputStream() { - - @Override - public int available() throws IOException { - return buff.remaining(); - } - - @Override - public int read() throws IOException { - if (buff.hasRemaining()) - return buff.get() & 0xFF; - - return -1; - } - - @Override - public int read(byte[] b) throws IOException { - if (!buff.hasRemaining()) - return -1; - - int len = b.length; - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, 0, len); - return len; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (len > buff.remaining()) - len = buff.remaining(); - buff.get(b, off, len); - return len; - } - - @Override - public long skip(long n) throws IOException { - if (!buff.hasRemaining()) - return -1; - - if (n > buff.remaining()) - n = buff.remaining(); - - buff.position(buff.position() + (int) n); - return n; - } - }; - } - - @Override - public long getContentLength() { - return resources.total; - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - outstream.write(resources.buffer, 0, resources.total); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java deleted file mode 100644 index ee093adc6fc..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Logger; - -/** - * A utility parent for searchers which gets data from web services which is incorporated into the query. - * This searcher will take care of implementing the search method while the extending class implements - * {@link #getQueryMap} and {@link #handleResponse} to create the http request and handle the response, respectively. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, is this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - - * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPClientSearcher extends HTTPSearcher { - - static final CompoundName REQUEST_META_CARRIER = new CompoundName("com.yahoo.search.federation.http.HTTPClientSearcher_requestMeta"); - - protected final static Logger log = Logger.getLogger(HTTPClientSearcher.class.getName()); - - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param path the path portion of the url to be used - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics) { - super(id, connections, path, statistics); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics, - CertificateStore certificateStore) { - super(id, connections, path, statistics, certificateStore); - } - - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, Statistics statistics) { - super(id, connections, parameters, statistics); - } - /** - * Creates a client searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - * @param certificateStore the certificate store to use to pass certificates in requests - */ - public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - } - - /** Overridden to avoid interfering with errors from nested searchers, which is inappropriate for a <i>client</i> */ - @Override - public Result robustSearch(Query query, Execution execution, Connection connection) { - return search(query,execution,connection); - } - - /** Implements a search towards the connection chosen by the cluster searcher for this query */ - @Override - public Result search(Query query, Execution execution, Connection connection) { - Hit requestMeta = doHttpRequest(query, connection); - Result result = execution.search(query); - result.hits().add(requestMeta); - return result; - } - - private Hit doHttpRequest(Query query, Connection connection) { - URI uri; - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - query.properties().set(REQUEST_META_CARRIER, requestMeta); - - query.trace("Created request information hit",false,9); - try { - uri = getURI(query, connection); - } catch (MalformedURLException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } catch (URISyntaxException e) { - query.errors().add(createMalformedUrlError(query,e)); - return requestMeta; - } - - HttpEntity entity; - try { - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - entity = getEntity(uri, requestMeta, query); - } catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this + " using " + connection + " for " + - query + ": " + Exceptions.toMessageString(e))); - return requestMeta; - } catch (TimeoutException e) { - query.errors().add(ErrorMessage.createTimeout("HTTP traffic timed out in " - + this + " for " + query + ": " + e.getMessage())); - return requestMeta; - } - if (entity==null) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "No result from connecting to HTTP backend in " + this + " using " + connection + " for " + query)); - return requestMeta; - } - - try { - query = handleResponse(entity,query); - } - catch (IOException e) { - query.errors().add(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - } finally { - cleanupHttpEntity(entity); - } - return requestMeta; - } - - /** Overrides to pass the query on to the next searcher */ - @Override - public Result search(Query query, Execution execution, ErrorMessage error) { - query.errors().add(error); - return execution.search(query); - } - - /** Do nothing on fill in client searchers */ - @Override - public void fill(Result result,String summaryClass,Execution execution,Connection connection) { - } - - /** - * Convenience hook for unmarshalling the response and adding the information to the query. - * Implement this or <code>handleResponse(entity,query)</code> in any subclass. - * This default implementation throws an exception. - * - * @param inputStream the stream containing the data from the http service - * @param contentLength the length of the content in the stream in bytes, or a negative number if not known - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - throw new UnsupportedOperationException("handleResponse must be implemented by " + this); - } - - /** - * Unmarshals the response and adds the resulting data to the given query. - * This default implementation calls - * <code>return handleResponse(entity.getContent(), entity.getContentLength(), query);</code> - * (and does some detailed query tracing). - * - * @param query the current query, to which information from the stream should be added - * @return query the query to propagate down the chain. This should almost always be the - * query instance given as a parameter. - */ - public Query handleResponse(HttpEntity entity, Query query) throws IOException { - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query = handleResponse(entity.getContent(), len, query); - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - return query; - } - - /** Never retry individual queries to clients for now */ - @Override - protected boolean shouldRetry(Query query,Result result) { return false; } - - /** - * numHits and offset should not be part of the cache key as cache supports - * partial read/write that is only one cache entry is maintained per query - * irrespective of the offset and numhits. - */ - public abstract Map<String, String> getCacheKey(Query q); - - /** - * Adds all key-values starting by "service." + getClientName() in query.properties(). - * Returns the empty map if {@link #getServiceName} is not overridden. - */ - @Override - public Map<String,String> getQueryMap(Query query) { - LinkedHashMap<String, String> queryMap=new LinkedHashMap<>(); - if (getServiceName().isEmpty()) return queryMap; - - for (Map.Entry<String,Object> objectProperty : query.properties().listProperties("service." + getServiceName()).entrySet()) // TODO: Make more efficient using CompoundName - queryMap.put(objectProperty.getKey(),objectProperty.getValue().toString()); - return queryMap; - } - - /** - * Override this to return the name of the service this is a client of. - * This is used to look up service specific properties as service.getServiceName.serviceSpecificProperty. - * This default implementation returns "", which means service specific parameters will not be used. - */ - protected String getServiceName() { return ""; } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java deleted file mode 100644 index a7b32678c1a..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.base.Preconditions; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnPerRouteBean; -import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.params.HttpParams; - -import com.yahoo.search.federation.ProviderConfig; - -/** - * A set of parameters for talking to an http backend - * - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public final class HTTPParameters { - - public static final String RETRIES = "com.yahoo.search.federation.http.retries"; - - private boolean frozen=false; - - // All timing parameters below are in milliseconds - /** The url request path portion */ - private String path="/"; - private int connectionTimeout=2000; - private int readTimeout=5000; - private boolean persistentConnections=true; - private boolean enableProxy = false; - private String proxyHost = "localhost"; - private int proxyPort = 1080; - private String method = "GET"; - private String schema = "http"; - private String inputEncoding = "utf-8"; - private String outputEncoding = "utf-8"; - private int maxTotalConnections=10000; - private int maxConnectionsPerRoute=10000; - private int socketBufferSizeBytes=-1; - private int retries = 1; - private int configuredReadTimeout = -1; - private int configuredConnectionTimeout = -1; - private int connectionPoolTimeout = -1; - private String certificateProxy = null; - private int certificatePort = 0; - private String certificateApplicationId = null; - private boolean certificateUseProxy = false; - private long certificateTtl = 0L; - private long certificateRetry = 0L; - - private PingOption.Enum pingOption = PingOption.NORMAL; - - - private boolean followRedirects = true; - - public HTTPParameters() {} - - public HTTPParameters(String path) { - setPath(path); - } - - public HTTPParameters(ProviderConfig providerConfig) { - configuredReadTimeout = (int) (providerConfig.readTimeout() * 1000.0d); - configuredConnectionTimeout = (int) (providerConfig.connectionTimeout() * 1000.0d); - connectionPoolTimeout = (int) (providerConfig.connectionPoolTimeout() * 1000.0d); - retries = providerConfig.retries(); - setPath(providerConfig.path()); - certificateUseProxy = providerConfig.yca().useProxy(); - if (certificateUseProxy) { - certificateProxy = providerConfig.yca().host(); - certificatePort = providerConfig.yca().port(); - } - certificateApplicationId = providerConfig.yca().applicationId(); - certificateTtl = providerConfig.yca().ttl() * 1000L; - certificateRetry = providerConfig.yca().retry() * 1000L; - followRedirects = providerConfig.followRedirects(); - pingOption = providerConfig.pingOption(); - } - - /** - * Set the url path to use in queries to this. If the argument is null or empty the path is set to "/". - * If a leading "/" is missing, it is added automatically. - */ - public final void setPath(String path) { - if (path==null || path.isEmpty()) path="/"; - - if (! path.startsWith("/")) - path="/" + path; - this.path = path; - } - - public PingOption.Enum getPingOption() { - return pingOption; - } - - public void setPingOption(PingOption.Enum pingOption) { - Preconditions.checkNotNull(pingOption); - ensureNotFrozen(); - this.pingOption = pingOption; - } - - /** Returns the url path. Default is "/". */ - public String getPath() { return path; } - - public boolean getFollowRedirects() { - return followRedirects; - } - - public void setFollowRedirects(boolean followRedirects) { - ensureNotFrozen(); - this.followRedirects = followRedirects; - } - - - public void setConnectionTimeout(int connectionTimeout) { - ensureNotFrozen(); - this.connectionTimeout=connectionTimeout; - } - - /** Returns the connection timeout in milliseconds. Default is 2000. */ - public int getConnectionTimeout() { return connectionTimeout; } - - public void setReadTimeout(int readTimeout) { - ensureNotFrozen(); - this.readTimeout=readTimeout; - } - - /** Returns the read timeout in milliseconds. Default is 5000. */ - public int getReadTimeout() { return readTimeout; } - - /** - * <b>Note: This is currently largely a noop: Connections are reused even when this is set to true. - * The setting will change from sharing connections between threads to only reusing it within a thread - * but it is still reused.</b> - */ - public void setPersistentConnections(boolean persistentConnections) { - ensureNotFrozen(); - this.persistentConnections=persistentConnections; - } - - /** Returns whether this should use persistent connections. Default is true. */ - public boolean getPersistentConnections() { return persistentConnections; } - - /** Returns whether proxying should be enabled. Default is false. */ - public boolean getEnableProxy() { return enableProxy; } - - public void setEnableProxy(boolean enableProxy ) { - ensureNotFrozen(); - this.enableProxy=enableProxy; - } - - /** Returns the proxy type to use (if enabled). Default is "http". */ - public String getProxyType() { - return "http"; - } - - public void setProxyHost(String proxyHost) { - ensureNotFrozen(); - this.proxyHost=proxyHost; - } - - /** Returns the proxy host to use (if enabled). Default is "localhost". */ - public String getProxyHost() { return proxyHost; } - - public void setProxyPort(int proxyPort) { - ensureNotFrozen(); - this.proxyPort=proxyPort; - } - - /** Returns the proxy port to use (if enabled). Default is 1080. */ - public int getProxyPort() { return proxyPort; } - - public void setMethod(String method) { - ensureNotFrozen(); - this.method=method; - } - - /** Returns the http method to use. Default is "GET". */ - public String getMethod() { return method; } - - public void setSchema(String schema) { - ensureNotFrozen(); - this.schema=schema; - } - - /** Returns the schema to use. Default is "http". */ - public String getSchema() { return schema; } - - public void setInputEncoding(String inputEncoding) { - ensureNotFrozen(); - this.inputEncoding=inputEncoding; - } - - /** Returns the input encoding. Default is "utf-8". */ - public String getInputEncoding() { return inputEncoding; } - - public void setOutputEncoding(String outputEncoding) { - ensureNotFrozen(); - this.outputEncoding=outputEncoding; - } - - /** Returns the output encoding. Default is "utf-8". */ - public String getOutputEncoding() { return outputEncoding; } - - /** Make this unmodifiable. Note that any thread synchronization must be done outside this object. */ - public void freeze() { - frozen=true; - } - - private void ensureNotFrozen() { - if (frozen) throw new IllegalStateException("Cannot modify frozen " + this); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams() { - return toHttpParams(connectionTimeout, readTimeout); - } - - /** - * Returns the eligible subset of this as a HttpParams snapshot - * AND configures the Apache HTTP library with the parameters of this - */ - public HttpParams toHttpParams(int connectionTimeout, int readTimeout) { - HttpParams params = new BasicHttpParams(); - // force use of configured value if available - if (configuredConnectionTimeout > 0) { - HttpConnectionParams.setConnectionTimeout(params, configuredConnectionTimeout); - } else { - HttpConnectionParams.setConnectionTimeout(params, connectionTimeout); - } - if (configuredReadTimeout > 0) { - HttpConnectionParams.setSoTimeout(params, configuredReadTimeout); - } else { - HttpConnectionParams.setSoTimeout(params, readTimeout); - } - if (socketBufferSizeBytes > 0) { - HttpConnectionParams.setSocketBufferSize(params, socketBufferSizeBytes); - } - if (connectionPoolTimeout > 0) { - ConnManagerParams.setTimeout(params, connectionPoolTimeout); - } - ConnManagerParams.setMaxTotalConnections(params, maxTotalConnections); - ConnManagerParams.setMaxConnectionsPerRoute(params, new ConnPerRouteBean(maxConnectionsPerRoute)); - if (retries >= 0) { - params.setIntParameter(RETRIES, retries); - } - params.setParameter("http.protocol.handle-redirects", followRedirects); - return params; - } - - public int getMaxTotalConnections() { - return maxTotalConnections; - } - - public void setMaxTotalConnections(int maxTotalConnections) { - ensureNotFrozen(); - this.maxTotalConnections = maxTotalConnections; - } - - public int getMaxConnectionsPerRoute() { - return maxConnectionsPerRoute; - } - - public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) { - ensureNotFrozen(); - this.maxConnectionsPerRoute = maxConnectionsPerRoute; - } - - public int getSocketBufferSizeBytes() { - return socketBufferSizeBytes; - } - - public void setSocketBufferSizeBytes(int socketBufferSizeBytes) { - ensureNotFrozen(); - this.socketBufferSizeBytes = socketBufferSizeBytes; - } - - public int getRetries() { - return retries; - } - - public void setRetries(int retries) { - ensureNotFrozen(); - this.retries = retries; - } - - public String getYcaProxy() { - return certificateProxy; - } - - public int getYcaPort() { - return certificatePort; - } - - public String getYcaApplicationId() { - return certificateApplicationId; - } - - public boolean getYcaUseProxy() { - return certificateUseProxy; - } - - public long getYcaTtl() { - return certificateTtl; - } - - public long getYcaRetry() { - return certificateRetry; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java deleted file mode 100644 index c72c2f26a1c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.common.collect.ImmutableList; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.federation.FederationSearcher; -import com.yahoo.search.query.Properties; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.statistics.Value; - -import org.apache.http.HttpEntity; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.*; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Superclass of searchers which talks to HTTP backends. Implement a subclass to talk to a backend - * over HTTP which is not supported by the platform out of the box. - * <p> - * Implementations must override one of the <code>unmarshal</code> methods to unmarshal the response. - * </p> - * - * @author Arne Bergene Fossaa - * @author bratseth - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPProviderSearcher extends HTTPSearcher { - - private final Counter emptyResults; - private final Value hitsPerQuery; - private final Value responseLatency; - private final Counter readTimeouts; - - private final static List<String> excludedSourceProperties = ImmutableList.of("offset", "hits", "provider"); - - protected final static Logger log = Logger.getLogger(HTTPProviderSearcher.class.getName()); - - /** The name of the cache used (which is just getid().stringValue(), or null if no cache is used */ - protected String cacheName=null; - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections,String path, Statistics statistics) { - this(id,connections,new HTTPParameters(path), statistics); - } - - /** Creates a http provider searcher using id.getName as provider name */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, String path, - Statistics statistics, CertificateStore certificateStore) { - this(id, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(id, connections, parameters, statistics, new ThrowingCertificateStore()); - } - - /** - * Creates a provider searcher - * - * @param id the id of this instance - * @param connections the connections this will load balance and fail over between - * @param parameters the parameters to use when making http calls - */ - public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(id, connections, parameters, statistics, certificateStore); - String suffix = "_" + getId().getName().replace('.', '_'); - hitsPerQuery = new Value("hits_per_query" + suffix, statistics, - new Value.Parameters().setLogRaw(false).setNameExtension(false).setLogMean(true)); - responseLatency = new Value(LOG_LATENCY_START + suffix, statistics, - new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false)); - emptyResults = new Counter("empty_results" + suffix, statistics, false); - readTimeouts = new Counter(LOG_READ_TIMEOUT_PREFIX + suffix, statistics, false); - } - - /** @deprecated this method does nothing */ - @Deprecated // OK - protected void configureCache(final QrBinaryCacheConfig cacheConfig,final QrBinaryCacheRegionConfig regionConfig) { - } - - /** - * Unmarshal the stream by converting it to hits and adding the hits to the given result. - * A convenience hook called by the default <code>unmarshal(entity,result).</code> - * Override this in subclasses which does not override <code>unmarshal(entity,result).</code> - * <p> - * This default implementation throws an exception. - * - * @param stream the stream of data returned - * @param contentLength the length of the content in bytes if known, or a negative number if unknown - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { - throw new UnsupportedOperationException("Unmarshal must be implemented by " + this); - } - - /** - * Unmarshal the result from an http entity. This default implementation calls - * <code>unmarshal(entity.getContent(), entity.getContentLength(), result)</code> - * (and does some detailed query tracing). - * - * @param entity the entity containing the data to unmarshal - * @param result the result to which unmarshalled data should be added - */ - public void unmarshal(HttpEntity entity,Result result) throws IOException { - Query query=result.getQuery(); - long len = entity.getContentLength(); - if (query.getTraceLevel()>=4) - query.trace("Received " + len + " bytes response in " + this, false, 4); - query.trace("Unmarshaling result.", false, 6); - unmarshal(entity.getContent(), len, result); - - if (query.getTraceLevel()>=2) - query.trace("Handled " + len + " bytes response in " + this, false, 2); - - } - - protected void addNonExcludedSourceProperties(Query query, Map<String, String> queryMap) { - Properties sourceProperties = FederationSearcher.getSourceProperties(query); - if (sourceProperties != null) { - for(Map.Entry<String, Object> entry : sourceProperties.listProperties("").entrySet()) { - if (!excludedSourceProperties.contains(entry.getKey())) { - queryMap.put(entry.getKey(), entry.getValue().toString()); - } - } - } - } - - /** - * Hook called at the moment the result is returned from this searcher. This default implementation - * does <code>return result</code>. - * - * @param result the result which is to be returned - * @param requestMeta the request information hit, or null if none was created (e.g if this was a cache lookup) - * @param e the exception caused during execution of this query, or null if none - * @return the result which is returned upwards - */ - protected Result inspectAndReturnFinalResult(Result result, Hit requestMeta, Exception e) { - return result; - } - - private Result statisticsBeforeInspection(Result result, Hit requestMeta, Exception e) { - int hitCount = result.getConcreteHitCount(); - if (hitCount == 0) { - emptyResults.increment(); - } - hitsPerQuery.put((double) hitCount); - - if (requestMeta != null) { - requestMeta.setField(LOG_HITCOUNT, Integer.valueOf(hitCount)); - } - - return inspectAndReturnFinalResult(result, requestMeta, e); - } - - - @Override - protected void logResponseLatency(long latency) { - responseLatency.put((double) latency); - } - - @Override - public Result search(Query query, Execution execution,Connection connection) { - // Create default meta hit for holding logging information - Hit requestMeta = createRequestMeta(); - Result result = new Result(query); - result.hits().add(requestMeta); - query.trace("Created request information hit", false, 9); - - try { - URI uri = getURI(query, requestMeta, connection); - if (query.getTraceLevel()>=1) - query.trace("Fetching " + uri.toString(), false, 1); - long requestStartTime = System.currentTimeMillis(); - - HttpEntity entity = getEntity(uri, requestMeta, query); - - // Why should consumeEntity call inspectAndReturnFinalResult itself? - // Seems confusing to me. - return entity == null - ? statisticsBeforeInspection(result, requestMeta, null) - : consumeEntity(entity, query, result, requestMeta, requestStartTime); - - } catch (MalformedURLException|URISyntaxException e) { - result.hits().addError(createMalformedUrlError(query,e)); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (TimeoutException e) { - result.hits().addError(ErrorMessage.createTimeout("No time left for HTTP traffic in " - + this - + " for " + query + ": " + e.getMessage())); - return statisticsBeforeInspection(result, requestMeta, e); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to connect to HTTP backend in " + this - + " for " + query + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, requestMeta, e); - } - } - - private Result consumeEntity(HttpEntity entity, Query query, Result result, Hit logHit, long requestStartTime) { - - try { - // remove some time from timeout to allow for close calls with return result - unmarshal(new TimedHttpEntity(entity, query.getStartTime(), Math.max(1, query.getTimeout() - 10)), result); - logHit.setField(LOG_LATENCY_FINISH, System.currentTimeMillis() - requestStartTime); - return statisticsBeforeInspection(result, logHit, null); - } catch (IOException e) { - result.hits().addError(ErrorMessage.createBackendCommunicationError( - "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e))); - return statisticsBeforeInspection(result, logHit, e); - } catch (TimeoutException e) { - readTimeouts.increment(); - result.hits().addError(ErrorMessage - .createTimeout("Timed out while reading/unmarshaling from backend in " - + this + " for " + query - + ": " + e.getMessage())); - return statisticsBeforeInspection(result, logHit, e); - } finally { // TODO: The scope of this finally must be enlarged to release the connection also on errors - cleanupHttpEntity(entity); - } - } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns the query.properties() prefixed by - * "source.[sourceName]" or "property.[propertyName]" - * (by calling {@link #addNonExcludedSourceProperties}). - */ - @Override - public Map<String,String> getQueryMap(Query query) { - Map<String,String> queryMap = super.getQueryMap(query); - addNonExcludedSourceProperties(query, queryMap); - return queryMap; - } - - /** - * @deprecated the cache key is ignored as there is no built-in caching support - */ - @Deprecated // OK - public abstract Map<String, String> getCacheKey(Query q); - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java deleted file mode 100644 index edf347bd84e..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java +++ /dev/null @@ -1,961 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.google.inject.Inject; -import com.yahoo.component.ComponentId; -import com.yahoo.jdisc.http.CertificateStore; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.yolean.Exceptions; -import com.yahoo.search.Query; -import com.yahoo.search.cluster.ClusterSearcher; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.statistics.Counter; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; - -import org.apache.http.*; -import org.apache.http.client.HttpClient; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ClientConnectionManager; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.conn.params.ConnManagerParams; -import org.apache.http.conn.params.ConnRoutePNames; -import org.apache.http.conn.routing.HttpRoutePlanner; -import org.apache.http.conn.scheme.PlainSocketFactory; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.scheme.SchemeRegistry; -import org.apache.http.conn.ssl.SSLSocketFactory; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.conn.DefaultHttpRoutePlanner; -import org.apache.http.impl.conn.SingleClientConnManager; -import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; -import org.apache.http.params.HttpParams; -import org.apache.http.params.HttpProtocolParams; -import org.apache.http.protocol.BasicHttpContext; -import org.apache.http.protocol.ExecutionContext; -import org.apache.http.protocol.HttpContext; -import org.apache.http.protocol.HttpRequestExecutor; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.SSLHandshakeException; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.io.UnsupportedEncodingException; -import java.net.*; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Generic superclass of searchers making connections to some HTTP service. This - * supports clustered connections - a list of alternative servers may be given, - * requests will be hashed across these and failed over in case some are down. - * <p> - * This simply provides some utility methods for working with http connections - * and implements ping against the service. - * - * <p>This searcher contains code from the Apache httpcomponents client library, - * licensed to the Apache Software Foundation under the Apache License, Version - * 2.0. Please refer to http://www.apache.org/licenses/LICENSE-2.0 for details. - * - * <p>This class automatically adds a meta hit containing latency and other - * meta information about the obtained HTTP data using createRequestMeta(). - * The fields available in the hit are:</p> - * - * <dl><dt> - * HTTPSearcher.LOG_LATENCY_START - * <dd> - * The latency of the external provider answering a request. - * <dt> - * HTTPSearcher.LOG_LATENCY_FINISH - * <dd> - * Total time of the HTTP traffic, but also decoding of the data, as this - * happens at the same time. - * <dt> - * HTTPSearcher.LOG_HITCOUNT - * <dd> - * Number of concrete hits in the result returned by this provider. - * <dt> - * HTTPSearcher.LOG_URI - * <dd> - * The complete URI used for external service. - * <dt> - * HTTPSearcher.LOG_SCHEME - * <dd> - * The scheme of the request URI sent. - * <dt> - * HTTPSearcher.LOG_HOST - * <dd> - * The host used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PORT - * <dd> - * The port used for the request URI sent. - * <dt> - * HTTPSearcher.LOG_PATH - * <dd> - * Path element of the request URI sent. - * <dt> - * HTTPSearcher.LOG_STATUS - * <dd> - * Status code of the HTTP response. - * <dt> - * HTTPSearcher.LOG_PROXY_TYPE - * <dd> - * The proxy type used, if any. Default is "http". - * <dt> - * HTTPSearcher.LOG_PROXY_HOST - * <dd> - * The proxy host, if any. - * <dt> - * HTTPSearcher.LOG_PROXY_PORT - * <dd> - * The proxy port, if any. - * <dt> - * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name - * <dd> - * The content of any additional request header fields. - * <dt> - * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name - * <dd> - * The content of any additional response header fields. - * </dl> - * - * @author Arne Bergene Fossaa - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public abstract class HTTPSearcher extends ClusterSearcher<Connection> { - - protected static final String YCA_HTTP_HEADER = "Yahoo-App-Auth"; - - private static final Charset iso8859Charset = Charset.forName("ISO-8859-1"); - - // Logging field name constants - public static final String LOG_PATH = "path"; - public static final String LOG_PORT = "port"; - public static final String LOG_HOST = "host"; - public static final String LOG_IP_ADDRESS = "ip_address"; - public static final String IP_ADDRESS_UNKNOWN = "unknown"; - - public static final String LOG_SCHEME = "scheme"; - public static final String LOG_URI = "uri"; - public static final String LOG_PROXY_PORT = "proxy_port"; - public static final String LOG_PROXY_HOST = "proxy_host"; - public static final String LOG_PROXY_TYPE = "proxy_type"; - public static final String LOG_STATUS = "status"; - public static final String LOG_LATENCY_FINISH = "latency_finish"; - public static final String LOG_LATENCY_START = "latency_start"; - public static final String LOG_LATENCY_CONNECT = "latency_connect"; - public static final String LOG_QUERY_PARAM_PREFIX = "query_param_"; - public static final String LOG_HEADER_PREFIX = "header_"; - public static final String LOG_RESPONSE_HEADER_PREFIX = "response_header_"; - public static final String LOG_HITCOUNT = "hit_count"; - public static final String LOG_CONNECT_TIMEOUT_PREFIX = "connect_timeout_"; - public static final String LOG_READ_TIMEOUT_PREFIX = "read_timeout_"; - - protected final Logger log = Logger.getLogger(HTTPSearcher.class.getName()); - - /** The HTTP parameters to use. Assigned in the constructor */ - private HTTPParameters httpParameters; - - private final Counter connectTimeouts; - - /** Whether to use certificates */ - protected boolean useCertificate = false; - - private final CertificateStore certificateStore; - - /** The (optional) certificate application ID. */ - private String certificateApplicationId = null; - - /** The (optional) certificate server proxy */ - protected HttpHost certificateProxy = null; - - /** Certificate cache TTL in ms */ - private long certificateTtl = 0L; - - /** Certificate server retry rate in the cache if no cert is found, in ms */ - private long certificateRetry = 0L; - - /** Set at construction if this is using persistent connections */ - private ClientConnectionManager sharedConnectionManager = null; - - /** Set at construction if using non-persistent connections */ - private ThreadLocal<SingleClientConnManager> singleClientConnManagerThreadLocal = null; - - private static final SchemeRegistry schemeRegistry = new SchemeRegistry(); - - static { - schemeRegistry.register(new Scheme("http", PlainSocketFactory - .getSocketFactory(), 80)); - schemeRegistry.register(new Scheme("https", SSLSocketFactory - .getSocketFactory(), 443)); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics) { - this(componentId, connections, new HTTPParameters(path), statistics, new ThrowingCertificateStore()); - } - - /** Creates a http searcher with default connection and read timeouts (currently 2 and 5s respectively) */ - public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics, - CertificateStore certificateStore) { - this(componentId, connections, new HTTPParameters(path), statistics, certificateStore); - } - - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics) { - this(componentId, connections, parameters, statistics, new ThrowingCertificateStore()); - } - /** - * Creates a http searcher - * - * @param componentId the id of this instance - * @param connections the connections to establish to the backend nodes - * @param parameters the http parameters to use. This object will be frozen if it isn't already - */ - @Inject - public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters, - Statistics statistics, CertificateStore certificateStore) { - super(componentId,connections,false); - String suffix = "_" + getId().getName().replace('.', '_'); - - connectTimeouts = new Counter(LOG_CONNECT_TIMEOUT_PREFIX + suffix, statistics, false); - - parameters.freeze(); - this.httpParameters = parameters; - this.certificateStore = certificateStore; - - if (parameters.getPersistentConnections()) { - HttpParams params=parameters.toHttpParams(); - HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1); - ConnManagerParams.setTimeout(params, 10); - sharedConnectionManager = new ThreadSafeClientConnManager(params, schemeRegistry); - Thread connectionPurgerThread = new Thread(() -> { - //this is the default value in yahoo jvm installations - long DNSTTLSec = 120; - while (true) { - try { - Thread.sleep(DNSTTLSec * 1000); - if (sharedConnectionManager == null) - continue; - - sharedConnectionManager.closeExpiredConnections(); - DNSTTLSec = Long.valueOf(java.security.Security - .getProperty("networkaddress.cache.ttl")); - //No DNS TTL, no need to close idle connections - if (DNSTTLSec <= 0) { - DNSTTLSec = 120; - continue; - } - sharedConnectionManager.closeIdleConnections(2 * DNSTTLSec, TimeUnit.SECONDS); - } catch (InterruptedException e) { - return; - } catch (NumberFormatException e) { - continue; - } - } - }); - connectionPurgerThread.setDaemon(true); - connectionPurgerThread.start(); - - } - else { - singleClientConnManagerThreadLocal =new ThreadLocal<>(); - } - - initializeCertificate(httpParameters, certificateStore); - } - - /** - * Initialize certificate store and proxy if they have been set to non-null, - * non-empty values. It will wrap thrown exceptions from the certificate store into - * RuntimeException and propagate them. - */ - private void initializeCertificate(HTTPParameters parameters, CertificateStore certificateStore) { - String applicationId = parameters.getYcaApplicationId(); - String proxy = parameters.getYcaProxy(); - int port = parameters.getYcaPort(); - long ttl = parameters.getYcaTtl(); - long retry = parameters.getYcaRetry(); - - if (applicationId != null && !applicationId.trim().isEmpty()) { - initializeCertificate(applicationId, ttl, retry, certificateStore); - } - - if (parameters.getYcaUseProxy()) { - initializeProxy(proxy, port); - } - } - - /** Returns the HTTP parameters used in this. This is always frozen */ - public HTTPParameters getParameters() { return httpParameters; } - - /** - * Returns the key-value pairs that should be added as properties to the request url sent to the service. - * Must be overridden in subclasses to add the key-values expected by the service in question, unless - * {@link #getURI} (from which this is called) is overridden. - * <p> - * This default implementation returns an empty LinkedHashMap. - */ - public Map<String,String> getQueryMap(Query query) { - return new LinkedHashMap<>(); - } - - /** - * Initialize the certificate. - * This will warn but not throw if certificates could not be loaded, as the certificates - * are external state which can fail independently. - */ - private void initializeCertificate(String applicationId, long ttl, long retry, CertificateStore certificateStore) { - try { - // get the certificate, i.e. init the cache and check integrity - String certificate = certificateStore.getCertificate(applicationId, ttl, retry); - if (certificate == null) { - getLogger().log(LogLevel.WARNING, "No certificate found for application '" + applicationId + "'"); - return; - } - - this.useCertificate = true; - this.certificateApplicationId = applicationId; - this.certificateTtl = ttl; - this.certificateRetry = retry; - getLogger().log(LogLevel.CONFIG, "Got certificate: " + certificate); - } - catch (Exception e) { - getLogger().log(LogLevel.WARNING,"Exception while initializing certificate for application '" + - applicationId + "' in " + this, e); - } - } - - /** - * Initialize the certificate proxy setting. - */ - private void initializeProxy(String host, int port) { - certificateProxy = new HttpHost(host, port); - getLogger().log(LogLevel.CONFIG, "Proxy is configured; will use proxy: " + certificateProxy); - } - - /** - * Same a {@code getURI(query, offset, hits, null)}. - * @see #getURI(Query, Hit, Connection) - */ - protected URI getURI(Query query,Connection connection) throws MalformedURLException, URISyntaxException { - Hit requestMeta; - try { - requestMeta = (Hit) query.properties().get(HTTPClientSearcher.REQUEST_META_CARRIER); - } catch (ClassCastException e) { - requestMeta = null; - } - return getURI(query, requestMeta, connection); - } - - /** - * Creates the URI for a query. - * Populates the {@code requestMeta} meta hit with the created URI HTTP properties. - * - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - */ - protected URI getURI(Query query, Hit requestMeta, Connection connection) - throws MalformedURLException, URISyntaxException { - StringBuilder parameters = new StringBuilder(); - - Map<String, String> queries = getQueryMap(query); - if (queries.size() > 0) { - Iterator<Map.Entry<String, String>> mapIterator = queries.entrySet().iterator(); - parameters.append("?"); - try { - Map.Entry<String, String> entry; - while (mapIterator.hasNext()) { - entry = mapIterator.next(); - - if (requestMeta != null) - requestMeta.setField(LOG_QUERY_PARAM_PREFIX - + entry.getKey(), entry.getValue()); - - parameters.append(entry.getKey() + "=" + URLEncoder.encode(entry.getValue(), - httpParameters.getInputEncoding())); - if (mapIterator.hasNext()) { - parameters.append("&"); - } - } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Unknown input encoding set in " + this, e); - } - } - - URI uri = new URL(httpParameters.getSchema(), connection.getHost(), - connection.getPort(), getPath() + parameters.toString()).toURI(); - if (requestMeta != null) { - requestMeta.setField(LOG_URI, uri.toString()); - requestMeta.setField(LOG_SCHEME, uri.getScheme()); - requestMeta.setField(LOG_HOST, uri.getHost()); - requestMeta.setField(LOG_PORT, uri.getPort()); - requestMeta.setField(LOG_PATH, uri.getPath()); - } - return uri; - } - - /** - * Called by getURI() to get the path of the URI for the external service. - * The default implementation returns httpParameters.getPath(); subclasses - * which only wants to override the path from httpParameters may use this - * method instead of overriding all of getURI(). - * - * @return the path to use for getURI - */ - protected String getPath() { - return httpParameters.getPath(); - } - - /** - * The URI that is used to check if the provider is up or down. This will again be used in the - * checkPing method by checking that we get a response that has a good status code (below 300). If better - * validation than just status code checking is needed, override the checkPing method. - */ - protected URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException { - return new URL(httpParameters.getSchema(),connection.getHost(),connection.getPort(),getPingPath()).toURI(); - } - - /** - * Called by getPingURI() to get the path of the URI for pinging the - * external service. The default implementation returns - * httpParameters.getPath(); subclasses which only wants to override the - * path from httpParameters may use this method instead of overriding all of - * getPingURI(). - * - * @return the path to use for getPingURI - */ - protected String getPingPath() { - return httpParameters.getPath(); - } - - /** - * Checks if the response is valid. - * @param response The response from the ping request - * @param pong The pong result to return back to the calling method. This method - * will add an error to the pong result (using addError) if the status of the HTTP response is 300 or above. - */ - protected void checkPing(HttpResponse response, Pong pong) { - if (response.getStatusLine().getStatusCode() >= 300) { - pong.addError(com.yahoo.search.result.ErrorMessage.createBackendCommunicationError( - "Got error " + response.getStatusLine().getStatusCode() - + " when contacting backend") - ); - } - } - - /** - * Pinging in HTTPBackend is done by creating a PING uri from http://host:port/path. - * If this returns a status that is below 300, the ping is considered good. - * - * If another uri is needed for pinging, reimplement getPingURI. - * - * Override either this method to change how ping - */ - @Override - public Pong ping(Ping ping, Connection connection) { - URI uri = null; - Pong pong = new Pong(); - HttpResponse response = null; - - if (httpParameters.getPingOption() == PingOption.DISABLE) - return pong; - - try { - uri = getPingURI(connection); - if (uri == null) - pong.addError(ErrorMessage.createIllegalQuery("Ping uri is null")); - if (uri.getHost()==null) { - pong.addError(ErrorMessage.createIllegalQuery("Ping uri has no host")); - uri=null; - } - } catch (MalformedURLException | URISyntaxException e) { - pong.addError(ErrorMessage.createIllegalQuery("Malformed ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + - " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } - - if (uri == null) return pong; - pong.setPingInfo("using uri '" + uri + "'"); - - try { - response = getPingResponse(uri, ping); - checkPing(response, pong); - } catch (IOException e) { - // We do not have a valid ping - pong.addError(ErrorMessage.createBackendCommunicationError( - "Exception thrown when pinging with url '" + uri + "': " + Exceptions.toMessageString(e))); - } catch (TimeoutException e) { - pong.addError(ErrorMessage.createTimeout("Timeout for ping " + uri + " in " + this + ": " + e.getMessage())); - } catch (RuntimeException e) { - log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + " using uri '" + uri + "'",e); - pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " + - Exceptions.toMessageString(e))); - } finally { - if (response != null) { - cleanupHttpEntity(response.getEntity()); - } - } - - return pong; - } - - private HttpResponse getPingResponse(URI uri, Ping ping) throws IOException { - long timeLeft = ping.getTimeout(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - - Map<String, String> requestHeaders = null; - if (httpParameters.getPingOption() == PingOption.YCA) - requestHeaders = generateYCAHeaders(); - - return getResponse(uri, null, requestHeaders, null, connectionTimeout, readTimeout); - } - - /** - * Same a {@code getEntity(uri, null)}. - * @param uri resource to fetch - * @param query the originating query - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Query query) throws IOException{ - return getEntity(uri, null, query); - } - - - /** - * Gets the HTTP entity that holds the response contents. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query - * @return the http entity, or null if none - * @throws java.io.IOException Whenever HTTP status code is in the 300 or higher range. - * @throws TimeoutException If query.timeLeft() equal to or lower than 0 - */ - protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { - if (query.getTimeLeft() <= 0) { - throw new TimeoutException("No time left for querying external backend."); - } - HttpResponse response = getResponse(uri, requestMeta, query); - StatusLine statusLine = response.getStatusLine(); - - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_STATUS, statusLine.getStatusCode()); - for (HeaderIterator headers = response.headerIterator(); headers.hasNext(); ) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_RESPONSE_HEADER_PREFIX + h.getName(), h.getValue()); - } - } - - if (statusLine.getStatusCode() >= 300) { - HttpEntity entity = response.getEntity(); - String message = createServerReporterErrorMessage(statusLine, entity); - cleanupHttpEntity(response.getEntity()); - throw new IOException(message); - } - - return response.getEntity(); - } - - private String createServerReporterErrorMessage(StatusLine statusLine, HttpEntity entity) { - String message = "Error when trying to connect to HTTP backend: " - + statusLine.getStatusCode() + " : " + statusLine.getReasonPhrase(); - - try { - if (entity != null) { - message += "(Message = " + EntityUtils.toString(entity) + ")"; - } - } catch (Exception e) { - log.log(LogLevel.WARNING, "Could not get message.", e); - } - - return message; - } - - /** - * Creates a meta hit dedicated to holding logging information. This hit has - * the 'logging:[searcher's ID]' type. - */ - protected Hit createRequestMeta() { - Hit requestMeta = new Hit("logging:" + getId().toString()); - requestMeta.setMeta(true); - requestMeta.types().add("logging"); - return requestMeta; - } - - protected void cleanupHttpEntity(HttpEntity entity) { - if (entity == null) return; - - try { - entity.consumeContent(); - } catch (IOException e) { - // It is ok if do not consume it, the resource will be freed after - // timeout. - // But log it just in case. - log.log(LogLevel.getVespaLogLevel(LogLevel.DEBUG), - "Not able to consume after processing: " + Exceptions.toMessageString(e)); - } - } - - /** - * Same as {@code getResponse(uri, null)}. - */ - protected HttpResponse getResponse(URI uri, Query query) throws IOException{ - return getResponse(uri, null, query); - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param query the originating query, used to calculate timeouts - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, Query query) throws IOException { - long timeLeft = query.getTimeLeft(); - int connectionTimeout = (int) (timeLeft / 4L); - int readTimeout = (int) (timeLeft * 3L / 4L); - connectionTimeout = connectionTimeout <= 0 ? 1 : connectionTimeout; - readTimeout = readTimeout <= 0 ? 1 : readTimeout; - HttpEntity reqEntity = getRequestEntity(query, requestMeta); - Map<String, String> reqHeaders = getRequestHeaders(query, requestMeta); - if ((reqEntity == null) && (reqHeaders == null)) { - return getResponse(uri, requestMeta, connectionTimeout, readTimeout); - } else { - return getResponse(uri, reqEntity, reqHeaders, requestMeta, connectionTimeout, readTimeout); - } - } - - /** - * Returns the set of headers to be passed in the http request to provider backend. The default - * implementation returns null, unless certificates are in use. If certificates are used, it will return a map - * only containing the needed certificate headers. - */ - protected Map<String, String> getRequestHeaders(Query query, Hit requestMeta) { - if (useCertificate) { - return generateYCAHeaders(); - } - return null; - } - - /** - * Returns the HTTP request entity to use when making the request for this query. - * This default implementation returns null. - * - * <p> Do return a repeatable entity if HTTP retry is active. - * - * @return the http request entity to use, or null to use the default entity - */ - protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { - return null; - } - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, Hit requestMeta, int connectionTimeout, int readTimeout) - throws IOException { - return getResponse(uri, null, null, requestMeta, connectionTimeout, readTimeout); - } - - - /** - * Executes an HTTP request and gets the response. - * @param uri the request URI. - * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}). - * @param connectionTimeout how long to wait for getting a connection - * @param readTimeout timeout for reading HTTP data - */ - protected HttpResponse getResponse(URI uri, HttpEntity reqEntity, - Map<String, String> reqHeaders, Hit requestMeta, - int connectionTimeout, int readTimeout) throws IOException { - - HttpParams httpParams = httpParameters.toHttpParams(connectionTimeout, readTimeout); - HttpClient httpClient = createClient(httpParams); - long start = 0L; - HttpUriRequest request; - if (httpParameters.getEnableProxy() && "http".equals(httpParameters.getProxyType())) { - HttpHost proxy = new HttpHost(httpParameters.getProxyHost(), - httpParameters.getProxyPort(), httpParameters.getProxyType()); - httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); - // Logging - if (requestMeta != null) { - requestMeta.setField(LOG_PROXY_TYPE, httpParameters.getProxyType()); - requestMeta.setField(LOG_PROXY_HOST, httpParameters.getProxyHost()); - requestMeta.setField(LOG_PROXY_PORT, httpParameters.getProxyPort()); - } - } - if (reqEntity == null) { - request = createRequest(httpParameters.getMethod(), uri); - } else { - request = createRequest(httpParameters.getMethod(), uri, reqEntity); - } - - if (reqHeaders != null) { - for (Entry<String, String> entry : reqHeaders.entrySet()) { - if (entry.getValue() == null || isAscii(entry.getValue())) { - request.addHeader(entry.getKey(), entry.getValue()); - } else { - byte[] asBytes = Utf8.toBytes(entry.getValue()); - String asLyingString = new String(asBytes, 0, asBytes.length, iso8859Charset); - request.addHeader(entry.getKey(), asLyingString); - } - } - } - - // Logging - if (requestMeta != null) { - for (HeaderIterator headers = request.headerIterator(); headers.hasNext();) { - Header h = headers.nextHeader(); - requestMeta.setField(LOG_HEADER_PREFIX + h.getName(), h.getValue()); - } - start = System.currentTimeMillis(); - } - - HttpResponse response; - - try { - HttpContext context = new BasicHttpContext(); - response = httpClient.execute(request, context); - - if (requestMeta != null) { - requestMeta.setField(LOG_IP_ADDRESS, getIpAddress(context)); - } - } catch (ConnectTimeoutException e) { - connectTimeouts.increment(); - throw e; - } - - // Logging - long latencyStart = System.currentTimeMillis() - start; - if (requestMeta != null) { - requestMeta.setField(LOG_LATENCY_START, latencyStart); - } - logResponseLatency(latencyStart); - return response; - } - - private String getIpAddress(HttpContext context) { - HttpConnection connection = (HttpConnection) context.getAttribute(ExecutionContext.HTTP_CONNECTION); - if (connection instanceof HttpInetConnection) { - InetAddress address = ((HttpInetConnection) connection).getRemoteAddress(); - String hostAddress = address.getHostAddress(); - return hostAddress == null ? - IP_ADDRESS_UNKNOWN: - hostAddress; - } else { - getLogger().log(LogLevel.DEBUG, "Unexpected connection type: " + connection.getClass().getName()); - return IP_ADDRESS_UNKNOWN; - } - } - - private boolean isAscii(String value) { - char[] scanBuffer = new char[value.length()]; - value.getChars(0, value.length(), scanBuffer, 0); - for (char c: scanBuffer) - if (c > 127) return false; - return true; - } - - protected void logResponseLatency(long latency) { } - - /** - * Creates a http client for one request. Override to customize the client - * to use, e.g for testing. This default implementation will add a certificate store - * proxy to params if is necessary, and then do - * <code>return new SearcherHttpClient(getConnectionManager(params), params);</code> - */ - protected HttpClient createClient(HttpParams params) { - if (certificateProxy != null) { - params.setParameter(ConnRoutePNames.DEFAULT_PROXY, certificateProxy); - } - return new SearcherHttpClient(getConnectionManager(params), params); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri) { - return createRequest(method, uri, null); - } - - /** - * Creates a HttpRequest. Override to customize the request. - * This default implementation does <code>return new HttpRequest(method,uri);</code> - */ - protected HttpUriRequest createRequest(String method,URI uri, HttpEntity entity) { - return new SearcherHttpRequest(method,uri); - } - - /** Get a connection manager which may be used safely from this thread */ - protected ClientConnectionManager getConnectionManager(HttpParams params) { - if (sharedConnectionManager != null) {// We are using shared connections - return sharedConnectionManager; - } else { - SingleClientConnManager singleClientConnManager = singleClientConnManagerThreadLocal.get(); - if (singleClientConnManager == null) { - singleClientConnManager = new SingleClientConnManager(params, schemeRegistry); - singleClientConnManagerThreadLocal.set(singleClientConnManager); - } - return singleClientConnManager; - } - } - - /** Utility method for creating error messages when a url is incorrect */ - protected ErrorMessage createMalformedUrlError(Query query,Exception e) { - return ErrorMessage.createErrorInPluginSearcher("Malformed url in " + this + " for " + query + - ": " + Exceptions.toMessageString(e)); - } - - private Map<String, String> generateYCAHeaders() { - Map<String, String> headers = new HashMap<>(); - String certificate = certificateStore.getCertificate(certificateApplicationId, certificateTtl, certificateRetry); - headers.put(YCA_HTTP_HEADER, certificate); - return headers; - } - - protected static class SearcherHttpClient extends DefaultHttpClient { - - private final int retries; - - public SearcherHttpClient(final ClientConnectionManager conman, final HttpParams params) { - super(conman, params); - retries = params.getIntParameter(HTTPParameters.RETRIES, 1); - addRequestInterceptor((request, context) -> { - if (!request.containsHeader("Accept-Encoding")) { - request.addHeader("Accept-Encoding", "gzip"); - } - }); - addResponseInterceptor((response, context) -> { - HttpEntity entity = response.getEntity(); - if (entity == null) return; - Header ceheader = entity.getContentEncoding(); - if (ceheader == null) return; - for (HeaderElement codec : ceheader.getElements()) { - if (codec.getName().equalsIgnoreCase("gzip")) { - response.setEntity(new GzipDecompressingEntity(response.getEntity())); - return; - } - } - }); - } - - @Override - protected HttpRequestExecutor createRequestExecutor() { - return new HttpRequestExecutor(); - } - - @Override - protected HttpRoutePlanner createHttpRoutePlanner() { - return new DefaultHttpRoutePlanner(getConnectionManager().getSchemeRegistry()); - } - - @Override - protected HttpRequestRetryHandler createHttpRequestRetryHandler() { - return new SearcherHttpRequestRetryHandler(retries); - } - } - - /** A retry handler which avoids retrying forever on errors misclassified as transient */ - private static class SearcherHttpRequestRetryHandler implements HttpRequestRetryHandler { - private final int retries; - - public SearcherHttpRequestRetryHandler(int retries) { - this.retries = retries; - } - - @Override - public boolean retryRequest(IOException e, int executionCount, HttpContext httpContext) { - if (e == null) { - throw new IllegalArgumentException("Exception parameter may not be null"); - } - if (executionCount > retries) { - return false; - } - if (e instanceof NoHttpResponseException) { - // Retry if the server dropped connection on us - return true; - } - if (e instanceof InterruptedIOException) { - // Timeout from federation layer - return false; - } - if (e instanceof UnknownHostException) { - // Unknown host - return false; - } - if (e instanceof SSLHandshakeException) { - // SSL handshake exception - return false; - } - return true; - } - - - } - - private static class SearcherHttpRequest extends HttpRequestBase { - String method; - - public SearcherHttpRequest(String method, final URI uri) { - super(); - this.method = method; - setURI(uri); - } - - @Override - public String getMethod() { - return method; - } - } - - /** - * Only for testing. - */ - public void shutdownConnectionManagers() { - ClientConnectionManager manager; - if (sharedConnectionManager != null) { - manager = sharedConnectionManager; - } else { - manager = singleClientConnManagerThreadLocal.get(); - } - if (manager != null) { - manager.shutdown(); - } - } - - protected static final class ThrowingCertificateStore implements CertificateStore { - - @Override - public String getCertificate(String key, long ttl, long retry) { - throw new UnsupportedOperationException("A certificate store is not available"); - } - - } - -} - diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java deleted file mode 100644 index 03ffe2b8a9c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -import org.apache.http.Header; -import org.apache.http.HttpEntity; - -/** - * Wrapper for adding timeout to an HttpEntity instance. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedHttpEntity implements HttpEntity { - /** - * The wrapped entity. Never null. - */ - private final HttpEntity entity; - private final long startTime; - private final long timeout; - - public TimedHttpEntity(HttpEntity entity, long startTime, long timeout) { - if (entity == null) { - throw new IllegalArgumentException("TimedHttpEntity cannot be instantiated with null HttpEntity."); - } - this.entity = entity; - this.startTime = startTime; - this.timeout = timeout; - } - - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - InputStream content = entity.getContent(); - if (content == null) { - return null; - } else { - return new TimedStream(content, startTime, timeout); - } - } - - - // START OF PURE FORWARDING METHODS - @Override - public void consumeContent() throws IOException { - entity.consumeContent(); - } - - - @Override - public Header getContentEncoding() { - return entity.getContentEncoding(); - } - - @Override - public long getContentLength() { - return entity.getContentLength(); - } - - @Override - public Header getContentType() { - return entity.getContentType(); - } - - @Override - public boolean isChunked() { - return entity.isChunked(); - } - - @Override - public boolean isRepeatable() { - return entity.isRepeatable(); - } - - @Override - public boolean isStreaming() { - return entity.isStreaming(); - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - entity.writeTo(outstream); - } - // END OF PURE FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java deleted file mode 100644 index 77a42ee0a34..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import java.io.IOException; -import java.io.InputStream; - -/** - * A stream which throws a TimeoutException if query timeout has been reached. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimedStream extends InputStream { - - /** - * A time barrier value, the point in time from which on read operations will cause an exception. - */ - private final long limit; - - /** - * A wrapped InputStream instance. - */ - private final InputStream content; - - /** - * Wrap an InputStream to make read operations potentially fire off - * TimeoutException. - * - * <p>Typical use would be<br> - * <code>new TimedStream(httpEntity.getContent(), query.getStartTime(), query.getTimeout())</code> - * - * @param content - * the InputStream to wrap - * @param startTime - * start time of query - * @param timeout - * how long the query is allowed to run - */ - public TimedStream(InputStream content, long startTime, long timeout) { - if (content == null) { - throw new IllegalArgumentException("Cannot instantiate TimedStream with null InputStream"); - } - this.content = content; - // The reasion for doing it in here instead of outside the constructor - // is this makes the usage of the class more intuitive IMHO - this.limit = startTime + timeout; - } - - private void checkTime(String message) { - if (System.currentTimeMillis() >= limit) { - throw new TimeoutException(message); - } - } - - // START FORWARDING METHODS: - // All methods below are forwarding methods to the contained stream, where - // some do a timeout check. - @Override - public int read() throws IOException { - int data = content.read(); - checkTime("Timed out during read()."); - return data; - } - - @Override - public int available() throws IOException { - return content.available(); - } - - @Override - public void close() throws IOException { - content.close(); - } - - @Override - public synchronized void mark(int readlimit) { - content.mark(readlimit); - } - - @Override - public boolean markSupported() { - return content.markSupported(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - int length = content.read(b, off, len); - checkTime("Timed out during read(byte[], int, int)"); - return length; - } - - @Override - public int read(byte[] b) throws IOException { - int length = content.read(b); - checkTime("Timed out during read(byte[])"); - return length; - } - - @Override - public synchronized void reset() throws IOException { - content.reset(); - } - - @Override - public long skip(long n) throws IOException { - long skipped = content.skip(n); - checkTime("Timed out during skip(long)"); - return skipped; - } - // END FORWARDING METHODS - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java deleted file mode 100644 index 18dd59ec37b..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -/** - * Timeout marker for slow HTTP connections. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class TimeoutException extends RuntimeException { - - /** - * Auto-generated version ID. - */ - private static final long serialVersionUID = 7084147598258586559L; - - public TimeoutException(String message) { - super(message); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java deleted file mode 100644 index af334b35221..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.search.federation.http; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java deleted file mode 100644 index 3a0db9b76de..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import java.util.Iterator; - -import com.yahoo.prelude.query.*; - -/** - * Marshal a query stack into an advanced query string suitable for - * passing to another QRS. - * - * @author Steinar Knutsen - * @author Rong-En Fan - * @deprecated use YQL - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -public class QueryMarshaller { - - private boolean atRoot = true; - - public String marshal(Item root) { - if (root == null || root instanceof NullItem) { - return null; - } - StringBuilder s = new StringBuilder(); - marshal(root, s); - atRoot = true; - return s.toString(); - } - - /** - * We do not yet care about exact match indices - */ - private void marshal(Item root, StringBuilder s) { - switch (root.getItemType()) { - case OR: - marshalOr((OrItem) root, s); - break; - case AND: - marshalAnd((CompositeItem) root, s); - break; - case NOT: - marshalNot((NotItem) root, s); - break; - case RANK: - marshalRank((RankItem) root, s); - break; - case WORD: - case INT: - case PREFIX: - case SUBSTRING: - case SUFFIX: - marshalWord((TermItem) root, s); - break; - case PHRASE: - // PhraseItem and PhraseSegmentItem don't add quotes for segmented - // termse - if (root instanceof PhraseSegmentItem) { - marshalPhrase((PhraseSegmentItem) root, s); - } else { - marshalPhrase((PhraseItem) root, s); - } - break; - case NEAR: - marshalNear((NearItem) root, s); - break; - case ONEAR: - marshalNear((ONearItem) root, s); - break; - case WEAK_AND: - marshalWeakAnd((WeakAndItem)root, s); - break; - default: - break; - } - } - - - private void marshalWord(TermItem item, StringBuilder s) { - String index = item.getIndexName(); - if (index.length() != 0) { - s.append(item.getIndexName()).append(':'); - } - s.append(item.stringValue()); - if (item.getWeight() != Item.DEFAULT_WEIGHT) - s.append("!").append(item.getWeight()); - } - - private void marshalRank(RankItem root, StringBuilder s) { - marshalComposite("RANK", root, s); - } - - private void marshalNot(NotItem root, StringBuilder s) { - marshalComposite("ANDNOT", root, s); - } - - private void marshalOr(OrItem root, StringBuilder s) { - marshalComposite("OR", root, s); - } - - /** - * Dump WORD items, and add space between each of them unless those - * words came from segmentation. - * - * @param root CompositeItem - * @param s current marshaled query - */ - private void dumpWords(CompositeItem root, StringBuilder s) { - for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) { - Item word = i.next(); - boolean useSeparator = true; - if (word instanceof TermItem) { - s.append(((TermItem) word).stringValue()); - if (word instanceof WordItem) { - useSeparator = !((WordItem) word).isFromSegmented(); - } - } else { - dumpWords((CompositeItem) word, s); - } - if (useSeparator && i.hasNext()) { - s.append(' '); - } - } - } - - private void marshalPhrase(PhraseItem root, StringBuilder s) { - marshalPhrase(root, s, root.isExplicit(), false); - } - - private void marshalPhrase(PhraseSegmentItem root, StringBuilder s) { - marshalPhrase(root, s, root.isExplicit(), true); - } - - private void marshalPhrase(IndexedItem root, StringBuilder s, boolean isExplicit, boolean isSegmented) { - String index = root.getIndexName(); - if (index.length() != 0) { - s.append(root.getIndexName()).append(':'); - } - if (isExplicit || !isSegmented) s.append('"'); - dumpWords((CompositeItem) root, s); - if (isExplicit || !isSegmented) s.append('"'); - } - - private void marshalNear(NearItem root, StringBuilder s) { - marshalComposite(root.getName() + "(" + root.getDistance() + ")", root, s); - } - - // Not only AndItem returns ItemType.AND - private void marshalAnd(CompositeItem root, StringBuilder s) { - marshalComposite("AND", root, s); - } - - private void marshalWeakAnd(WeakAndItem root, StringBuilder s) { - marshalComposite("WAND(" + root.getN() + ")", root, s); - } - - private void marshalComposite(String operator, CompositeItem root, StringBuilder s) { - boolean useParen = !atRoot; - if (useParen) { - s.append("( "); - } else { - atRoot = false; - } - for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) { - Item item = i.next(); - marshal(item, s); - if (i.hasNext()) - s.append(' ').append(operator).append(' '); - } - if (useParen) { - s.append(" )"); - } - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java deleted file mode 100644 index a6aec30b496..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java +++ /dev/null @@ -1,642 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.result.Relevance; -import com.yahoo.text.XML; -import org.xml.sax.*; -import org.xml.sax.helpers.DefaultHandler; -import org.xml.sax.helpers.XMLReaderFactory; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; - -import static com.yahoo.text.Lowercase.toLowerCase; - -/** - * Parse Vespa XML results and create Result instances. - * - * @author Steinar Knutsen - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated // OK -@SuppressWarnings("deprecation") -public class ResultBuilder extends DefaultHandler { - private static final String ERROR = "error"; - - private static final String FIELD = "field"; - - private static Logger log = Logger.getLogger(ResultBuilder.class.getName()); - - /** Namespaces feature id (http://xml.org/sax/features/namespaces). */ - protected static final String NAMESPACES_FEATURE_ID = "http://xml.org/sax/features/namespaces"; - - /** - * Namespace prefixes feature id - * (http://xml.org/sax/features/namespace-prefixes). - */ - protected static final String NAMESPACE_PREFIXES_FEATURE_ID = "http://xml.org/sax/features/namespace-prefixes"; - - /** Validation feature id (http://xml.org/sax/features/validation). */ - protected static final String VALIDATION_FEATURE_ID = "http://xml.org/sax/features/validation"; - - /** - * Schema validation feature id - * (http://apache.org/xml/features/validation/schema). - */ - protected static final String SCHEMA_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/schema"; - - /** - * Dynamic validation feature id - * (http://apache.org/xml/features/validation/dynamic). - */ - protected static final String DYNAMIC_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/dynamic"; - - // default settings - - /** Default parser name. */ - protected static final String DEFAULT_PARSER_NAME = "org.apache.xerces.parsers.SAXParser"; - - /** Default namespaces support (false). */ - protected static final boolean DEFAULT_NAMESPACES = false; - - /** Default namespace prefixes (false). */ - protected static final boolean DEFAULT_NAMESPACE_PREFIXES = false; - - /** Default validation support (false). */ - protected static final boolean DEFAULT_VALIDATION = false; - - /** Default Schema validation support (false). */ - protected static final boolean DEFAULT_SCHEMA_VALIDATION = false; - - /** Default dynamic validation support (false). */ - protected static final boolean DEFAULT_DYNAMIC_VALIDATION = false; - - private StringBuilder fieldContent; - - private String fieldName; - - private int fieldLevel = 0; - - private boolean hasLiteralTags = false; - - private Map<String, Object> hitFields = new HashMap<>(); - private String hitType; - private String hitRelevance; - private String hitSource; - - private int offset = 0; - - private List<Tag> tagStack = new ArrayList<>(); - - private final XMLReader parser; - - private Query query; - - private Result result; - - private static enum ResultPart { - ROOT, ERRORDETAILS, HIT, HITGROUP; - } - - Deque<ResultPart> location = new ArrayDeque<>(10); - - private String currentErrorCode; - - private String currentError; - - private Deque<HitGroup> hitGroups = new ArrayDeque<>(5); - - private static class Tag { - public final String name; - - /** - * Offset is a number which is generated for all data and tags inside - * fields, used to determine whether a tag was closed without enclosing - * any characters or other tags. - */ - public final int offset; - - public Tag(final String name, final int offset) { - this.name = name; - this.offset = offset; - } - - @Override - public String toString() { - return name + '(' + Integer.valueOf(offset) + ')'; - } - } - - /** Default constructor. */ - public ResultBuilder() throws RuntimeException { - this(createParser()); - } - - public ResultBuilder(XMLReader parser) { - this.parser = parser; - this.parser.setContentHandler(this); - this.parser.setErrorHandler(this); - } - - public static XMLReader createParser() { - ClassLoader savedContextClassLoader = Thread.currentThread().getContextClassLoader(); - Thread.currentThread().setContextClassLoader(org.apache.xerces.parsers.SAXParser.class.getClassLoader()); - - try { - XMLReader reader = XMLReaderFactory.createXMLReader(DEFAULT_PARSER_NAME); - setParserFeatures(reader); - return reader; - } catch (Exception e) { - throw new RuntimeException("error: Unable to instantiate parser (" - + DEFAULT_PARSER_NAME + ")", e); - } finally { - Thread.currentThread().setContextClassLoader(savedContextClassLoader); - } - } - - private static void setParserFeatures(XMLReader reader) { - try { - reader.setFeature(NAMESPACES_FEATURE_ID, DEFAULT_NAMESPACES); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + NAMESPACES_FEATURE_ID + ")"); - } - try { - reader.setFeature(NAMESPACE_PREFIXES_FEATURE_ID, - DEFAULT_NAMESPACE_PREFIXES); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + NAMESPACE_PREFIXES_FEATURE_ID + ")"); - } - try { - reader.setFeature(VALIDATION_FEATURE_ID, DEFAULT_VALIDATION); - } catch (SAXException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + VALIDATION_FEATURE_ID + ")"); - } - try { - reader.setFeature(SCHEMA_VALIDATION_FEATURE_ID, - DEFAULT_SCHEMA_VALIDATION); - } catch (SAXNotRecognizedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not recognize feature (" - + SCHEMA_VALIDATION_FEATURE_ID + ")"); - - } catch (SAXNotSupportedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + SCHEMA_VALIDATION_FEATURE_ID + ")"); - } - - try { - reader.setFeature(DYNAMIC_VALIDATION_FEATURE_ID, - DEFAULT_DYNAMIC_VALIDATION); - } catch (SAXNotRecognizedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not recognize feature (" - + DYNAMIC_VALIDATION_FEATURE_ID + ")"); - - } catch (SAXNotSupportedException e) { - log.log(LogLevel.WARNING, "warning: Parser does not support feature (" - + DYNAMIC_VALIDATION_FEATURE_ID + ")"); - } - } - - @Override - public void startDocument() throws SAXException { - reset(); - result = new Result(query); - hitGroups.addFirst(result.hits()); - location.addFirst(ResultPart.ROOT); - return; - } - - private void reset() { - result = null; - fieldLevel = 0; - hasLiteralTags = false; - tagStack = null; - fieldContent = null; - offset = 0; - currentError = null; - currentErrorCode = null; - hitGroups.clear(); - location.clear(); - } - - @Override - public void startElement(String uri, String local, String raw, - Attributes attrs) throws SAXException { - // "Everybody" wants this switch to be moved into the - // enum class instead, but in this case, I find the classic - // approach more readable. - switch (location.peekFirst()) { - case HIT: - if (fieldLevel > 0) { - tagInField(raw, attrs, FIELD); - ++offset; - return; - } - if (FIELD.equals(raw)) { - ++fieldLevel; - fieldName = attrs.getValue("name"); - fieldContent = new StringBuilder(); - hasLiteralTags = false; - } - break; - case ERRORDETAILS: - if (fieldLevel > 0) { - tagInField(raw, attrs, ERROR); - ++offset; - return; - } - if (ERROR.equals(raw)) { - if (attrs != null) { - currentErrorCode = attrs.getValue("code"); - currentError = attrs.getValue("error"); - } - ++fieldLevel; - fieldContent = new StringBuilder(); - hasLiteralTags = false; - } - break; - case HITGROUP: - if ("hit".equals(raw)) { - startHit(attrs); - } else if ("group".equals(raw)) { - startHitGroup(attrs); - } - break; - case ROOT: - if ("hit".equals(raw)) { - startHit(attrs); - } else if ("errordetails".equals(raw)) { - location.addFirst(ResultPart.ERRORDETAILS); - } else if ("result".equals(raw)) { - if (attrs != null) { - String total = attrs.getValue("total-hit-count"); - if (total != null) { - result.setTotalHitCount(Long.valueOf(total)); - } - } - } else if ("group".equals(raw)) { - startHitGroup(attrs); - } else if (ERROR.equals(raw)) { - if (attrs != null) { - currentErrorCode = attrs.getValue("code"); - fieldContent = new StringBuilder(); - } - } - break; - } - ++offset; - } - - private void startHitGroup(Attributes attrs) { - HitGroup g = new HitGroup(); - Set<String> types = g.types(); - - final String source; - if (attrs != null) { - String groupType = attrs.getValue("type"); - if (groupType != null) { - for (String s : groupType.split(" ")) { - if (s.length() > 0) { - types.add(s); - } - } - } - - source = attrs.getValue("source"); - } else { - source = null; - } - - g.setId((source != null) ? source : "dummy"); - - hitGroups.peekFirst().add(g); - hitGroups.addFirst(g); - location.addFirst(ResultPart.HITGROUP); - } - - private void startHit(Attributes attrs) { - hitFields.clear(); - location.addFirst(ResultPart.HIT); - if (attrs != null) { - hitRelevance = attrs.getValue("relevancy"); - hitSource = attrs.getValue("source"); - hitType = attrs.getValue("type"); - } else { - hitRelevance = null; - hitSource = null; - hitType = null; - } - } - - private void tagInField(String tag, Attributes attrs, String enclosingTag) { - if (!hasLiteralTags) { - hasLiteralTags = true; - String fieldTillNow = XML.xmlEscape(fieldContent.toString(), false); - fieldContent = new StringBuilder(fieldTillNow); - tagStack = new ArrayList<>(); - } - if (enclosingTag.equals(tag)) { - ++fieldLevel; - } - if (tagStack.size() > 0) { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (prevTag != null && (prevTag.offset + 1) == offset) { - fieldContent.append(">"); - } - } - fieldContent.append("<").append(tag); - if (attrs != null) { - int attrCount = attrs.getLength(); - for (int i = 0; i < attrCount; i++) { - fieldContent.append(" ").append(attrs.getQName(i)) - .append("=\"").append( - XML.xmlEscape(attrs.getValue(i), true)).append( - "\""); - } - } - tagStack.add(new Tag(tag, offset)); - } - - private void endElementInField(String qName, String enclosingTag) { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) { - fieldContent.append(" />"); - } else { - fieldContent.append("</").append(qName).append('>'); - } - if (prevTag.name.equals(qName)) { - tagStack.remove(tagStack.size() - 1); - } - } - - private void endElementInHitField(String qName) { - if (FIELD.equals(qName) && --fieldLevel == 0) { - Object content; - if (hasLiteralTags) { - content = new XMLString(fieldContent.toString()); - } else { - content = fieldContent.toString(); - } - hitFields.put(fieldName, content); - if ("collapseId".equals(fieldName)) { - hitFields.put(fieldName, Integer.valueOf(content.toString())); - } - fieldName = null; - fieldContent = null; - tagStack = null; - } else { - Tag prevTag = tagStack.get(tagStack.size() - 1); - if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) { - fieldContent.append(" />"); - } else { - fieldContent.append("</").append(qName).append('>'); - } - if (prevTag.name.equals(qName)) { - tagStack.remove(tagStack.size() - 1); - } - } - } - @Override - public void characters(char ch[], int start, int length) - throws SAXException { - - switch (location.peekFirst()) { - case ERRORDETAILS: - case HIT: - if (fieldLevel > 0) { - if (hasLiteralTags) { - if (tagStack.size() > 0) { - Tag tag = tagStack.get(tagStack.size() - 1); - if (tag != null && (tag.offset + 1) == offset) { - fieldContent.append(">"); - } - } - fieldContent.append( - XML.xmlEscape(new String(ch, start, length), false)); - } else { - fieldContent.append(ch, start, length); - } - } - break; - default: - if (fieldContent != null) { - fieldContent.append(ch, start, length); - } - break; - } - ++offset; - } - - @Override - public void ignorableWhitespace(char ch[], int start, int length) - throws SAXException { - return; - } - - @Override - public void processingInstruction(String target, String data) - throws SAXException { - return; - } - - @Override - public void endElement(String namespaceURI, String localName, String qName) - throws SAXException { - switch (location.peekFirst()) { - case HITGROUP: - if ("group".equals(qName)) { - hitGroups.removeFirst(); - location.removeFirst(); - } - break; - case HIT: - if (fieldLevel > 0) { - endElementInHitField(qName); - } else if ("hit".equals(qName)) { - //assert(hitKeys.size() == hitValues.size()); - //We try to get either uri or documentID and use that as id - Object docId = extractDocumentID(); - Hit newHit = new Hit(docId.toString()); - if (hitRelevance != null) newHit.setRelevance(new Relevance(Double.parseDouble(hitRelevance))); - if(hitSource != null) newHit.setSource(hitSource); - if(hitType != null) { - for(String type: hitType.split(" ")) { - newHit.types().add(type); - } - } - for(Map.Entry<String, Object> field : hitFields.entrySet()) { - newHit.setField(field.getKey(), field.getValue()); - } - - hitGroups.peekFirst().add(newHit); - location.removeFirst(); - } - break; - case ERRORDETAILS: - if (fieldLevel == 1 && ERROR.equals(qName)) { - ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode), - currentError, - fieldContent.toString()); - hitGroups.peekFirst().addError(error); - currentError = null; - currentErrorCode = null; - fieldContent = null; - tagStack = null; - fieldLevel = 0; - } else if (fieldLevel > 0) { - endElementInField(qName, ERROR); - } else if ("errordetails".equals(qName)) { - location.removeFirst(); - } - break; - case ROOT: - if (ERROR.equals(qName)) { - ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode), - fieldContent.toString()); - hitGroups.peekFirst().addError(error); - currentErrorCode = null; - fieldContent = null; - } - break; - default: - break; - } - ++offset; - } - - private Object extractDocumentID() { - Object docId = null; - if (hitFields.containsKey("uri")) { - docId = hitFields.get("uri"); - } else { - final String documentId = "documentId"; - if (hitFields.containsKey(documentId)) { - docId = hitFields.get(documentId); - } else { - final String lcDocumentId = toLowerCase(documentId); - for (Map.Entry<String, Object> e : hitFields.entrySet()) { - String key = e.getKey(); - // case insensitive matching, checking length first hoping to avoid some lowercasing - if (documentId.length() == key.length() && lcDocumentId.equals(toLowerCase(key))) { - docId = e.getValue(); - break; - } - } - } - } - if (docId == null) { - docId = "dummy"; - log.info("Results from vespa backend did not contain either uri or documentId"); - } - return docId; - } - - @Override - public void warning(SAXParseException ex) throws SAXException { - printError("Warning", ex); - } - - @Override - public void error(SAXParseException ex) throws SAXException { - printError("Error", ex); - } - - @Override - public void fatalError(SAXParseException ex) throws SAXException { - printError("Fatal Error", ex); - // throw ex; - } - - /** Prints the error message. */ - protected void printError(String type, SAXParseException ex) { - StringBuilder errorMessage = new StringBuilder(); - - errorMessage.append(type); - if (ex != null) { - String systemId = ex.getSystemId(); - if (systemId != null) { - int index = systemId.lastIndexOf('/'); - if (index != -1) - systemId = systemId.substring(index + 1); - errorMessage.append(' ').append(systemId); - } - } - errorMessage.append(':') - .append(ex.getLineNumber()) - .append(':') - .append(ex.getColumnNumber()) - .append(": ") - .append(ex.getMessage()); - log.log(LogLevel.WARNING, errorMessage.toString()); - - } - - public Result parse(String identifier, Query query) { - Result toReturn; - - setQuery(query); - try { - parser.parse(identifier); - } catch (SAXParseException e) { - // ignore - } catch (Exception e) { - log.log(LogLevel.WARNING, "Error parsing result from Vespa",e); - Exception se = e; - if (e instanceof SAXException) { - se = ((SAXException) e).getException(); - } - if (se != null) - se.printStackTrace(System.err); - else - e.printStackTrace(System.err); - } - toReturn = result; - reset(); - return toReturn; - } - - public Result parse(InputSource input, Query query) { - Result toReturn; - - setQuery(query); - try { - parser.parse(input); - } catch (SAXParseException e) { - // ignore - } catch (Exception e) { - log.log(LogLevel.WARNING, "Error parsing result from Vespa",e); - Exception se = e; - if (e instanceof SAXException) { - se = ((SAXException) e).getException(); - } - if (se != null) - se.printStackTrace(System.err); - else - e.printStackTrace(System.err); - } - toReturn = result; - reset(); - return toReturn; - } - - - private void setQuery(Query query) { - this.query = query; - } -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java deleted file mode 100644 index 246732d0970..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java +++ /dev/null @@ -1,259 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa; - -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Map; -import java.util.Set; - -import com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher; -import org.xml.sax.InputSource; -import org.xml.sax.XMLReader; - -import com.google.inject.Inject; -import com.yahoo.collections.Tuple2; -import com.yahoo.component.ComponentId; -import com.yahoo.component.Version; -import com.yahoo.component.chain.dependencies.After; -import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.QueryCanonicalizer; -import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.cache.QrBinaryCacheConfig; -import com.yahoo.search.cache.QrBinaryCacheRegionConfig; -import com.yahoo.search.federation.ProviderConfig; -import com.yahoo.search.federation.http.Connection; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.textserialize.TextSerialize; -import com.yahoo.search.yql.MinimalQueryInserter; -import com.yahoo.statistics.Statistics; - -import edu.umd.cs.findbugs.annotations.Nullable; - -/** - * Backend searcher for external Vespa clusters (queried over http). - * - * <p>If the "sources" argument should be honored on an external cluster - * when using YQL+, override {@link #chooseYqlSources(Set)}.</p> - * - * @author Arne Bergene Fossaa - * @author Steinar Knutsen - * @deprecated - */ -// TODO: Remove on Vespa 7 -@Deprecated -@Provides("Vespa") -@After("*") -public class VespaSearcher extends ConfiguredHTTPProviderSearcher { - - private final ThreadLocal<XMLReader> readerHolder = new ThreadLocal<>(); - private final Query.Type queryType; - private final Tuple2<String, Version> segmenterVersion; - - private static final CompoundName select = new CompoundName("select"); - private static final CompoundName streamingUserid = new CompoundName("streaming.userid"); - private static final CompoundName streamingGroupname = new CompoundName("streaming.groupname"); - private static final CompoundName streamingSelection = new CompoundName("streaming.selection"); - - /** Create an instance from configuration */ - public VespaSearcher(ComponentId id, ProviderConfig config, QrBinaryCacheConfig c, - QrBinaryCacheRegionConfig r, Statistics statistics) { - this(id, config, c, r, statistics, null); - } - - /** - * Create an instance from configuration - * - * @param linguistics used for generating meta info for YQL+ - */ - @Inject - public VespaSearcher(ComponentId id, ProviderConfig config, - QrBinaryCacheConfig c, QrBinaryCacheRegionConfig r, - Statistics statistics, @Nullable Linguistics linguistics) { - super(id, config, c, r, statistics); - queryType = toQueryType(config.queryType()); - if (linguistics == null) { - segmenterVersion = null; - } else { - segmenterVersion = linguistics.getVersion(Linguistics.Component.SEGMENTER); - } - } - - /** - * Create an instance from direct parameters having a single connection. - * Useful for testing - */ - public VespaSearcher(String idString, String host, int port, String path) { - super(idString, host, port, path, Statistics.nullImplementation); - queryType = toQueryType(ProviderConfig.QueryType.LEGACY); - segmenterVersion = null; - } - - void addProperty(Map<String, String> queryMap, Query query, CompoundName property) { - Object o = query.properties().get(property); - if (o != null) { - queryMap.put(property.toString(), o.toString()); - } - } - - @Override - public Map<String, String> getQueryMap(Query query) { - Map<String, String> queryMap = getQueryMapWithoutHitsOffset(query); - queryMap.put("offset", Integer.toString(query.getOffset())); - queryMap.put("hits", Integer.toString(query.getHits())); - queryMap.put("presentation.format", "xml"); - - addProperty(queryMap, query, select); - addProperty(queryMap, query, streamingUserid); - addProperty(queryMap, query, streamingGroupname); - addProperty(queryMap, query, streamingSelection); - return queryMap; - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return getQueryMapWithoutHitsOffset(q); - } - - private Map<String, String> getQueryMapWithoutHitsOffset(Query query) { - Map<String, String> queryMap = super.getQueryMap(query); - if (queryType == Query.Type.YQL) { - queryMap.put(MinimalQueryInserter.YQL.toString(), marshalQuery(query)); - } else { - queryMap.put("query", marshalQuery(query.getModel().getQueryTree())); - queryMap.put("type", queryType.toString()); - } - - addNonExcludedSourceProperties(query, queryMap); - return queryMap; - } - - Query.Type toQueryType(ProviderConfig.QueryType.Enum providerQueryType) { - if (providerQueryType == ProviderConfig.QueryType.LEGACY) { - return Query.Type.ADVANCED; - } else if (providerQueryType == ProviderConfig.QueryType.PROGRAMMATIC) { - return Query.Type.PROGRAMMATIC; - } else if (providerQueryType == ProviderConfig.QueryType.YQL) { - return Query.Type.YQL; - } else if (providerQueryType == ProviderConfig.QueryType.SELECT) { - return Query.Type.SELECT; - } else { - throw new RuntimeException("Query type " + providerQueryType + " unsupported."); - } - } - - /** - * Serialize the query parameter for outgoing queries. For YQL+ queries, - * sources and fields will be set to all sources and all fields, to follow - * the behavior of other query types. - * - * @param query - * the current, outgoing query - * @return a string to include in an HTTP request - */ - public String marshalQuery(Query query) { - if (queryType != Query.Type.YQL) { - return marshalQuery(query.getModel().getQueryTree()); - } - - query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once - Query workQuery = query.clone(); - String error = QueryCanonicalizer.canonicalize(workQuery); - if (error != null) { - getLogger().log(LogLevel.WARNING,"Could not normalize [" + query.toString() + "]: " + error); - // Just returning null here is the pattern from existing code... - return null; - } - chooseYqlSources(workQuery.getModel().getSources()); - chooseYqlSummaryFields(workQuery.getPresentation().getSummaryFields()); - return workQuery.yqlRepresentation(getSegmenterVersion(), false); - } - - public String marshalQuery(QueryTree root) { - QueryTree rootClone = root.clone(); // TODO: Why? - String error = QueryCanonicalizer.canonicalize(rootClone); - if (error != null) return null; - - return marshalRoot(rootClone.getRoot()); - } - - private String marshalRoot(Item root) { - switch (queryType) { - case ADVANCED: return new QueryMarshaller().marshal(root); - case PROGRAMMATIC: return TextSerialize.serialize(root); - default: throw new RuntimeException("Unsupported query type."); - } - } - - private XMLReader getReader() { - XMLReader reader = readerHolder.get(); - if (reader == null) { - reader = ResultBuilder.createParser(); - readerHolder.set(reader); - } - return reader; - } - - @Override - public void unmarshal(InputStream stream, long contentLength, Result result) { - ResultBuilder parser = new ResultBuilder(getReader()); - Result mResult = parser.parse(new InputSource(stream), - result.getQuery()); - result.mergeWith(mResult); - result.hits().addAll(mResult.hits().asUnorderedHits()); - } - - /** Returns the canonical Vespa ping URI, http://host:port/status.html */ - @Override - public URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException { - return new URL(getParameters().getSchema(), connection.getHost(), - connection.getPort(), "/status.html").toURI(); - } - - /** - * Get the segmenter version data used when creating YQL queries. Useful if - * overriding {@link #marshalQuery(Query)}. - * - * @return a tuple with the name of the segmenting engine in use, and its - * version - */ - protected Tuple2<String, Version> getSegmenterVersion() { - return segmenterVersion; - } - - /** - * Choose which source arguments to use for the external cluster when - * generating a YQL+ query string. This is called from - * {@link #marshalQuery(Query)}. The default implementation clears the set, - * i.e. requests all sources. Other implementations may modify the source - * set as they see fit, or simply do nothing. - * - * @param sources - * the set of source names to use for the outgoing query - */ - protected void chooseYqlSources(Set<String> sources) { - sources.clear(); - } - - /** - * Choose which summary fields to request from the external cluster when - * generating a YQL+ query string. This is called from - * {@link #marshalQuery(Query)}. The default implementation clears the set, - * i.e. requests all fields. Other implementations may modify the summary - * field set as they see fit, or simply do nothing. - * - * @param summaryFields - * the set of source names to use for the outgoing query - */ - protected void chooseYqlSummaryFields(Set<String> summaryFields) { - summaryFields.clear(); - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java deleted file mode 100644 index a1c9236ff9c..00000000000 --- a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.search.federation.vespa; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java deleted file mode 100644 index a679b17b6fc..00000000000 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.searchchain.model.federation; - -import com.yahoo.container.bundle.BundleInstantiationSpecification; -import net.jcip.annotations.Immutable; - -import com.yahoo.search.federation.http.HTTPProviderSearcher; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Specifies how a http provider is to be set up. - * - * @author Tony Vaagenes - * @deprecated - */ -@Immutable -@Deprecated // OK -// TODO: Remove on Vespa 7 -public class HttpProviderSpec { - - public enum Type { - vespa(com.yahoo.search.federation.vespa.VespaSearcher.class); - - Type(Class<? extends HTTPProviderSearcher> searcherClass) { - className = searcherClass.getName(); - } - - final String className; - } - - // The default connection parameter values come from the config definition - public static class ConnectionParameters { - public final Double readTimeout; - public final Double connectionTimeout; - public final Double connectionPoolTimeout; - public final Integer retries; - - public ConnectionParameters(Double readTimeout, Double connectionTimeout, - Double connectionPoolTimeout, Integer retries) { - this.readTimeout = readTimeout; - this.connectionTimeout = connectionTimeout; - this.connectionPoolTimeout = connectionPoolTimeout; - this.retries = retries; - } - } - - public static class Node { - public final String host; - public final int port; - - public Node(String host, int port) { - this.host = host; - this.port = port; - } - - @Override - public String toString() { - return "Node{" + - "host='" + host + '\'' + - ", port=" + port + - '}'; - } - } - - public final ConnectionParameters connectionParameters; - - public final Integer cacheSizeMB; - - public final String path; - public final List<Node> nodes; - public final String ycaApplicationId; - public final Integer ycaCertificateTtl; - public final Integer ycaRetryWait; - public final Node ycaProxy; - - public static BundleInstantiationSpecification toBundleInstantiationSpecification(Type type) { - return BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(type.className, null); - } - - public static boolean includesType(String typeString) { - for (Type type : Type.values()) { - if (type.name().equals(typeString)) { - return true; - } - } - return false; - } - - public HttpProviderSpec(Double cacheWeight, - String path, - List<Node> nodes, - String ycaApplicationId, - Integer ycaCertificateTtl, - Integer ycaRetryWait, - Node ycaProxy, - Integer cacheSizeMB, - ConnectionParameters connectionParameters) { - - this.path = path; - this.nodes = unmodifiable(nodes); - this.ycaApplicationId = ycaApplicationId; - this.ycaProxy = ycaProxy; - this.ycaCertificateTtl = ycaCertificateTtl; - this.ycaRetryWait = ycaRetryWait; - this.cacheSizeMB = cacheSizeMB; - - this.connectionParameters = connectionParameters; - } - - private List<HttpProviderSpec.Node> unmodifiable(List<HttpProviderSpec.Node> nodes) { - return nodes == null ? - Collections.emptyList() : - Collections.unmodifiableList(new ArrayList<>(nodes)); - } -} diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java deleted file mode 100644 index f86d76a12a4..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java +++ /dev/null @@ -1,309 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.templates.test; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.io.IOUtils; -import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.prelude.templates.SearchRendererAdaptor; -import com.yahoo.prelude.templates.TiledTemplateSet; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.federation.http.HTTPProviderSearcher; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.nio.charset.CharsetDecoder; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Tests representing a federated and grouped result as a Result object and - * rendering a tiled output of the result - * - * @author bratseth - */ -@SuppressWarnings("deprecation") -public class TilingTestCase { - - /** - * This result contains two blocks (center and right). - * The center block contains multiple subblocks while the right one contains a single block of ads. - */ - @Test - public void testTiling() throws IOException { - Chain<Searcher> chain=new Chain<>("tiling", new TiledResultProducer()); - - // Query it - Query query = new Query("/tiled?query=foo"); - Result result = callSearchAndSetRenderer(chain, query); - assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml")),result); - } - - /** - * This result contains center section and meta blocks. - */ - @Test - public void testTiling2() throws IOException { - Chain<Searcher> chain= new Chain<>("tiling", new TiledResultProducer2()); - - // Query it - Query query=new Query("/tiled?query=foo"); - Result result= callSearchAndSetRenderer(chain, query); - assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml")),result); - } - - private Result callSearchAndSetRenderer(Chain<Searcher> chain, Query query) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - Result result = new Execution(chain, context).search(query); - result.getTemplating().setRenderer(new SearchRendererAdaptor(new TiledTemplateSet())); - return result; - } - - public static void assertRenderedStartsWith(String expected,Result result) throws IOException { - assertRendered(expected,result,false); - } - - public static void assertRendered(String expected,Result result) throws IOException { - assertRendered(expected,result,true); - } - - public static void assertRendered(String expected, Result result,boolean checkFullEquality) throws IOException { - if (checkFullEquality) - assertEquals(filterComments(expected), getRendered(result)); - else - assertTrue(getRendered(result).startsWith(expected)); - } - - private static String filterComments(String s) { - StringBuilder b = new StringBuilder(); - for (String line : s.split("\\\n")) - if ( ! line.startsWith("<!--")) - b.append(line).append("\n"); - return b.toString(); - } - - public static String getRendered(Result result) throws IOException { - if (result.getTemplating().getRenderer() == null) - result.getTemplating().setTemplates(null); - - // Done in a roundabout way to simulate production code path - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - Charset cs = Charset.forName(result.getTemplating().getRenderer().getEncoding()); - CharsetDecoder decoder = cs.newDecoder(); - SearchRendererAdaptor.callRender(stream, result); - stream.flush(); - return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString(); - } - - private static class TiledResultProducer extends Searcher { - - @Override - public Result search(Query query, Execution execution) { - Result result = new Result(query); - result.setTotalHitCount(2800000000l); - - // Blocks - HitGroup centerBlock=(HitGroup)result.hits().add(new HitGroup("section:center")); - centerBlock.types().add("section"); - centerBlock.setField("region","center"); - - HitGroup rightBlock=(HitGroup)result.hits().add(new HitGroup("section:right")); - rightBlock.types().add("section"); - rightBlock.setField("region","right"); - - // Center groups - HitGroup navigation=(HitGroup)centerBlock.add(new HitGroup("group:navigation",1.0)); - navigation.types().add("group"); - navigation.setField("type","navigation"); - - HitGroup adsNorth=(HitGroup)centerBlock.add(new HitGroup("group:ads:north",0.9)); - adsNorth.types().add("group"); - adsNorth.setField("type","ads"); - - HitGroup hits=(HitGroup)centerBlock.add(new HitGroup("group:navigation",0.8)); - hits.types().add("group"); - hits.setField("type","hits"); - - HitGroup adsSouth=(HitGroup)centerBlock.add(new HitGroup("group:ads:south",0.7)); - adsSouth.types().add("group"); - adsSouth.setField("type","ads"); - - // Right group - HitGroup adsRight=(HitGroup)rightBlock.add(new HitGroup("group:ads:right",0.7)); - adsRight.types().add("group"); - adsRight.setField("type","ads"); - - // Navigation content - /* - Hit alsoTry=navigation.add(new Hit("alsotry")); - alsoTry.types().add("alsotry"); - alsoTry.setMeta(true); - LinkList links=new LinkList(); - links.add(new Link("Hilton","?search=Hilton hotel")); - links.add(new Link("Habbo hotel","?search=Habbo hotel")); - links.add(new Link("Marriott","?search=Marriott hotel")); - alsoTry.setField("links",links); - */ - - // North ads content - Hit ad1=adsNorth.add(new Hit("http://www.hotels.com",0.7)); - ad1.types().add("ad"); - ad1.setAuxiliary(true); - ad1.setField("title",new XMLString("Cheap <hi>hotels</hi>")); - ad1.setField("body",new XMLString("Low Rates Guaranteed. Call a <hi>Hotel</hi> Expert.")); - - Hit ad2=adsNorth.add(new Hit("http://www.expedia.com",0.6)); - ad2.types().add("ad"); - ad2.setAuxiliary(true); - ad2.setField("title",new XMLString("Cheap <hi>hotels</hi> at Expedia")); - ad2.setField("body","Expedia Special Rates Means We Guarantee Our Low Rates on Rooms."); - -// // Hits content -// // - news hit -// HitGroup news1=(HitGroup)hits.add(new HitGroup("newsarticles",0.9)); -// news1.setMeta(false); -// news1.types().add("news"); -// news1.setField("title","Hotel - News results"); -// Hit article1=news1.add(new Hit("www.miamiherald.com/?article=jhsgd7323",0.5)); -// article1.setAuxiliary(true); -// article1.setField("title","Celebrity blackout: The Hilton of Paris changes name to regain search traffic"); -// article1.types().add("newsarticle"); -// article1.setField("age",23); -// article1.setField("source","Miami Herald"); -// Hit article2=news1.add(new Hit("www.sfgate.com/?article=8763khj7",0.4)); -// article2.setAuxiliary(true); -// article2.setField("title","Hotels - more expensive than staying at home"); -// article2.types().add("newsarticle"); -// article2.setField("age",3500); -// article2.setField("source","SF Gate"); - - // - collapsed hit - Hit hit1=hits.add(new Hit("www.hotels.com",0.8)); - hit1.types().add("hit.collapsed"); - hit1.setField("title","Hotels.com | Cheap Hotels | Discount Hotel Rooms | Motels | Lodging"); - hit1.setField("body",new XMLString("Hotels.com helps you find great rates on hotels and discount <hi>hotel</hi> packages.")); - /* - LinkList collapsed1=new LinkList(); - collapsed1.add(new Link("Last Minute Deals","www.hotels.com/lastminutedeals")); - collapsed1.add(new Link("Hotel Savings","www.hotels.com/deals")); - collapsed1.add(new Link("800-994-6835","www.hotels.com/?PSRC=OT2")); - hit1.setField("links",collapsed1); - */ - - // regular hit with links - Hit hit2=hits.add(new Hit("www.indigohotels.com",0.7)); - hit2.types().add("hit"); - hit2.setField("title","Hotel Indigo Hotels United States - Official Web Site"); - hit2.setField("body","Make Hotel Indigo online hotel reservations and book your hotel rooms today."); - /* - LinkList collapsed2=new LinkList(); - collapsed2.add(new Link("800-333-6835","www.indigohotels.com/order")); - collapsed2.add(new Link("Reservations","www.indigohotels.com/reservations")); - hit2.setField("links",collapsed2); - */ - - // boring old hit - Hit hit3=hits.add(new Hit("www.all-hotels.com",0.6)); - hit3.types().add("hit"); - hit3.setField("title","All hotels"); - hit3.setField("body","Online hotel directory and reservations."); - - // South ads - Hit southAd1=adsSouth.add(new Hit("www.daysinn.com",1.0)); - southAd1.types().add("ad"); - southAd1.setAuxiliary(true); - southAd1.setField("title","Days Inn Special Deal"); - southAd1.setField("body","Buy now and Save 15% Off Our Best Available Rate with Days Inn."); - Hit southAd2=adsSouth.add(new Hit("http://www.expedia.com",0.9)); - southAd2.types().add("ad"); - southAd2.setAuxiliary(true); - southAd2.setField("title",new XMLString("Cheap <hi>hotels</hi> at Expedia")); - southAd2.setField("body","Expedia Special Rates Means We Guarantee Our Low Rates on Rooms."); - - // Right ads - Hit rightAd1=adsRight.add(new Hit("www.daysinn.com",1.0)); - rightAd1.types().add("ad"); - rightAd1.setAuxiliary(true); - rightAd1.setField("title","Days Inn Special Deal"); - rightAd1.setField("body","Buy now and Save 15% Off Our Best Available Rate with Days Inn."); - Hit rightAd2=adsRight.add(new Hit("www.holidayinn.com",0.9)); - rightAd2.types().add("ad"); - rightAd2.setAuxiliary(true); - rightAd2.setField("title","Holiday Inn: Official Site"); - rightAd2.setField("body","Book with Holiday Inn. Free Internet. Kids eat free."); - - // Done creating result - must analyze because we add ads then later set them as auxiliary - result.analyzeHits(); - - return result; - } - - } - - private static class TiledResultProducer2 extends Searcher { - - @Override - public Result search(Query query,Execution execution) { - Result result=new Result(query); - result.setTotalHitCount(1); - - HitGroup section = new HitGroup("section:center"); - result.hits().add(section); - section.setMeta(false); - section.types().add("section"); - section.setField("region", "center"); - - HitGroup yst = new HitGroup("yst"); - section.add(yst); - yst.setMeta(false); - yst.setSource("sr"); - yst.types().add("sr"); - yst.setField("provider", "yst"); - - Hit theHit = new Hit("159"); - yst.add(theHit); - theHit.setSource("sr"); - theHit.types().add("sr"); - theHit.setField("provider", "yst"); - theHit.setField("title", "Yahoo"); - - HitGroup meta = new HitGroup("meta"); - result.hits().add(meta); - meta.types().add("meta"); - - Hit log = new Hit("com.yahoo.search.federation.yst.YSTBackendSearcherproxy-tw1cache.idp.inktomisearch.com55556/search"); - meta.add(log); - log.setMeta(true); - log.setSource("sr"); - log.setField("provider", "yst"); - log.types().add("logging"); - log.setField(HTTPProviderSearcher.LOG_URI, "http://proxy-tw1cache.idp.inktomisearch.com:55556/search?qp=yahootw-twp&Fields=url%2Credirecturl%2Cdate%2Csize%2Cformat%2Csms_product%2Ccacheurl%2Cnodename%2Cid%2Clanguage%2Crsslinks%2Crssvalidatedlinks%2Ccpc%2Cclustertype%2Cxml.active_abstract%2Cactive_abstract_type%2Cactive_abstract_source%2Ccontract_id%2Ctranslated%2Cxml.ydir_tw_hotlist_data%2Cxml.summary%2Cclustercollision%2Cxml.pi_info%2Cpage_adult_overridable%2Cpage_spam_overridable%2Ccategory_ydir%2Chate_edb&Unique=doc%2Chost+2&QueryEncoding=utf-8&Query=ALLWORDS%28yahoo%29&Database=dewownrm-zh-tw&FirstResult=0&srcpvid=&cacheecho=1&ResultsEncoding=utf-8&QueryLanguage=Chinese-traditional&Region=US&NumResults=10&Client=yahoous2"); - log.setField(HTTPProviderSearcher.LOG_SCHEME, "http"); - log.setField(HTTPProviderSearcher.LOG_HOST, "proxy-tw1cache.idp.inktomisearch.com"); - log.setField(HTTPProviderSearcher.LOG_PORT, "55556"); - log.setField(HTTPProviderSearcher.LOG_PATH, "/search"); - log.setField(HTTPProviderSearcher.LOG_STATUS, "200"); - log.setField(HTTPProviderSearcher.LOG_LATENCY_CONNECT, "757"); - log.setField(HTTPProviderSearcher.LOG_RESPONSE_HEADER_PREFIX + "content-length", "16217"); - - result.analyzeHits(); - - return result; - } - - } - -} diff --git a/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java deleted file mode 100644 index 3e57c48e7f3..00000000000 --- a/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.prelude.test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import com.google.common.util.concurrent.MoreExecutors; -import com.yahoo.component.chain.Chain; -import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.templates.test.TilingTestCase; -import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.search.result.Hit; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.prelude.fastsearch.FastHit; -import com.yahoo.search.Searcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - -/** - * Tests that rank features are rendered when requested in the query - * - * @author bratseth - */ -@SuppressWarnings("deprecation") -public class RankFeatureDumpTestCase { - - private static final String rankFeatureString= - "{\"match.weight.as1\":10,\"attribute(ai1)\":1.000000,\"proximity(as1, 1, 2)\":2.000000}"; - - @Test - public void test() throws IOException { - Query query=new Query("?query=five&rankfeatures"); - assertTrue(query.getRanking().getListFeatures()); // New api - Result result = doSearch(new MockBackend(), query, 0,10); - assertTrue(TilingTestCase.getRendered(result).contains( - "<field name=\"" + com.yahoo.search.result.Hit.RANKFEATURES_FIELD + "\">" + rankFeatureString + "</field>")); - } - - private static class MockBackend extends Searcher { - - @Override - public Result search(com.yahoo.search.Query query, Execution execution) { - Result result=new Result(query); - Hit hit=new FastHit("test",1000); - hit.setField(com.yahoo.search.result.Hit.RANKFEATURES_FIELD,rankFeatureString); - result.hits().add(hit); - return result; - } - - } - - private Result doSearch(Searcher searcher, Query query, int offset, int hits) { - query.setOffset(offset); - query.setHits(hits); - return createExecution(searcher).search(query); - } - - private Execution createExecution(Searcher searcher) { - Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics()); - return new Execution(chainedAsSearchChain(searcher), context); - } - - private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { - List<Searcher> searchers = new ArrayList<>(); - searchers.add(topOfChain); - return new Chain<>(searchers); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java deleted file mode 100644 index 6babba5a36a..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import static org.junit.Assert.*; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.UnknownHostException; -import java.util.Arrays; -import java.util.Random; -import java.util.zip.GZIPOutputStream; - -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.message.BasicHeader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.yahoo.text.Utf8; - -/** - * Test GZip support for the HTTP integration introduced in 4.2. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class GzipDecompressingEntityTestCase { - private static final String STREAM_CONTENT = "00000000000000000000000000000000000000000000000000"; - private static final byte[] CONTENT_AS_BYTES = Utf8.toBytes(STREAM_CONTENT); - GzipDecompressingEntity testEntity; - - private static final class MockEntity implements HttpEntity { - - private final InputStream inStream; - - MockEntity(InputStream is) { - inStream = is; - } - - @Override - public boolean isRepeatable() { - return false; - } - - @Override - public boolean isChunked() { - return false; - } - - @Override - public long getContentLength() { - return -1; - } - - @Override - public Header getContentType() { - return new BasicHeader("Content-Type", "text/plain"); - } - - @Override - public Header getContentEncoding() { - return new BasicHeader("Content-Encoding", "gzip"); - } - - @Override - public InputStream getContent() throws IOException, - IllegalStateException { - return inStream; - } - - @Override - public void writeTo(OutputStream outstream) throws IOException { - } - - @Override - public boolean isStreaming() { - return false; - } - - @Override - public void consumeContent() throws IOException { - } - } - - @Before - public void setUp() throws Exception { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - GZIPOutputStream gzip = new GZIPOutputStream(out); - gzip.write(CONTENT_AS_BYTES); - gzip.finish(); - gzip.close(); - byte[] compressed = out.toByteArray(); - InputStream inStream = new ByteArrayInputStream(compressed); - testEntity = new GzipDecompressingEntity(new MockEntity(inStream)); - } - - @After - public void tearDown() throws Exception { - } - - @Test - public final void testGetContentLength() throws UnknownHostException { - assertEquals(STREAM_CONTENT.length(), testEntity.getContentLength()); - } - - @Test - public final void testGetContent() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length]; - int read = in.read(buffer); - assertEquals(CONTENT_AS_BYTES.length, read); - assertArrayEquals(CONTENT_AS_BYTES, buffer); - } - - @Test - public final void testGetContentToBigArray() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; - in.read(buffer); - byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); - assertArrayEquals(expected, buffer); - } - - @Test - public final void testGetContentAvailable() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - assertEquals(CONTENT_AS_BYTES.length, in.available()); - } - - @Test - public final void testLargeZip() throws IOException { - byte [] input = new byte [10000000]; - Random random = new Random(89); - for (int i = 0; i < input.length; i++) { - input[i] = (byte) random.nextInt(); - } - ByteArrayOutputStream out = new ByteArrayOutputStream(); - GZIPOutputStream gzip = new GZIPOutputStream(out); - gzip.write(input); - gzip.finish(); - gzip.close(); - byte[] compressed = out.toByteArray(); - assertEquals(10003073, compressed.length); - InputStream inStream = new ByteArrayInputStream(compressed); - GzipDecompressingEntity gunzipper = new GzipDecompressingEntity(new MockEntity(inStream)); - assertEquals(input.length, gunzipper.getContentLength()); - byte[] buffer = new byte[input.length]; - InputStream content = gunzipper.getContent(); - assertEquals(input.length, content.available()); - int read = content.read(buffer); - assertEquals(input.length, read); - assertArrayEquals(input, buffer); - } - - @Test - public final void testGetContentReadByte() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; - int i = 0; - while (i < buffer.length) { - int r = in.read(); - if (r == -1) { - break; - } else { - buffer[i++] = (byte) r; - } - } - byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2); - assertEquals(CONTENT_AS_BYTES.length, i); - assertArrayEquals(expected, buffer); - } - - @Test - public final void testGetContentReadWithOffset() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2]; - int read = in.read(buffer, CONTENT_AS_BYTES.length, CONTENT_AS_BYTES.length); - assertEquals(CONTENT_AS_BYTES.length, read); - byte[] expected = new byte[CONTENT_AS_BYTES.length * 2]; - for (int i = 0; i < CONTENT_AS_BYTES.length; ++i) { - expected[CONTENT_AS_BYTES.length + i] = CONTENT_AS_BYTES[i]; - } - assertArrayEquals(expected, buffer); - read = in.read(buffer, 0, CONTENT_AS_BYTES.length); - assertEquals(-1, read); - } - - @Test - public final void testGetContentSkip() throws IllegalStateException, IOException { - InputStream in = testEntity.getContent(); - final long n = 5L; - long skipped = in.skip(n); - assertEquals(n, skipped); - int read = in.read(); - assertEquals(CONTENT_AS_BYTES[(int) n], read); - skipped = in.skip(5000); - assertEquals(CONTENT_AS_BYTES.length - n - 1, skipped); - assertEquals(-1L, in.skip(1L)); - } - - - @Test - public final void testWriteToOutputStream() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - testEntity.writeTo(out); - assertArrayEquals(CONTENT_AS_BYTES, out.toByteArray()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java deleted file mode 100644 index 11f5d7d14fd..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.search.federation.ProviderConfig; -import org.junit.Test; - -import static com.yahoo.search.federation.ProviderConfig.Yca; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author gjoranv - * @author Steinar Knutsen - */ -public class HttpParametersTest { - - @Test - public void create_from_config() throws Exception { - ProviderConfig config = new ProviderConfig(new ProviderConfig.Builder() - .connectionTimeout(1.0) - .maxConnectionPerRoute(2) - .maxConnections(3) - .path("myPath") - .readTimeout(4) - .socketBufferBytes(5) - .yca(new Yca.Builder() - .applicationId("myId") - .host("myYcaHost") - .port(7) - .retry(8) - .ttl(9) - .useProxy(true))); - - HTTPParameters httpParameters = new HTTPParameters(config); - - // Written to configuredConnectionTimeout, but it is not accessible!? - //assertThat(httpParameters.getConnectionTimeout(), is(1000)); - - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getMaxConnectionsPerRoute(), is(2)); - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getMaxTotalConnections(), is(3)); - - assertThat(httpParameters.getPath(), is("/myPath")); - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getReadTimeout(), is(4)); - - // This value is not set from config by the constructor!? - //assertThat(httpParameters.getSocketBufferSizeBytes(), is(5)); - - - assertThat(httpParameters.getYcaUseProxy(), is(true)); - assertThat(httpParameters.getYcaApplicationId(), is("myId")); - assertThat(httpParameters.getYcaProxy(), is("myYcaHost")); - assertThat(httpParameters.getYcaPort(), is(7)); - assertThat(httpParameters.getYcaRetry(), is(8000L)); - assertThat(httpParameters.getYcaTtl(), is(9000L)); - } - - @Test - public void requireFreezeWorksForAccessors() { - HTTPParameters p = new HTTPParameters(); - boolean caught = false; - final int expected = 37; - p.setConnectionTimeout(expected); - assertEquals(expected, p.getConnectionTimeout()); - p.freeze(); - try { - p.setConnectionTimeout(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setReadTimeout(expected); - assertEquals(expected, p.getReadTimeout()); - p.freeze(); - try { - p.setReadTimeout(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setPersistentConnections(true); - assertTrue(p.getPersistentConnections()); - p.freeze(); - try { - p.setPersistentConnections(false); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - assertEquals("http", p.getProxyType()); - - p = new HTTPParameters(); - caught = false; - p.setEnableProxy(true); - assertTrue(p.getEnableProxy()); - p.freeze(); - try { - p.setEnableProxy(false); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setProxyHost("nalle"); - assertEquals("nalle", p.getProxyHost()); - p.freeze(); - try { - p.setProxyHost("jappe"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setProxyPort(expected); - assertEquals(expected, p.getProxyPort()); - p.freeze(); - try { - p.setProxyPort(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setMethod("POST"); - assertEquals("POST", p.getMethod()); - p.freeze(); - try { - p.setMethod("GET"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setSchema("gopher"); - assertEquals("gopher", p.getSchema()); - p.freeze(); - try { - p.setSchema("http"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setInputEncoding("iso-8859-15"); - assertEquals("iso-8859-15", p.getInputEncoding()); - p.freeze(); - try { - p.setInputEncoding("shift-jis"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setOutputEncoding("iso-8859-15"); - assertEquals("iso-8859-15", p.getOutputEncoding()); - p.freeze(); - try { - p.setOutputEncoding("shift-jis"); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setMaxTotalConnections(expected); - assertEquals(expected, p.getMaxTotalConnections()); - p.freeze(); - try { - p.setMaxTotalConnections(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setMaxConnectionsPerRoute(expected); - assertEquals(expected, p.getMaxConnectionsPerRoute()); - p.freeze(); - try { - p.setMaxConnectionsPerRoute(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setSocketBufferSizeBytes(expected); - assertEquals(expected, p.getSocketBufferSizeBytes()); - p.freeze(); - try { - p.setSocketBufferSizeBytes(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - - p = new HTTPParameters(); - caught = false; - p.setRetries(expected); - assertEquals(expected, p.getRetries()); - p.freeze(); - try { - p.setRetries(0); - } catch (IllegalStateException e) { - caught = true; - } - assertTrue(caught); - } -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java deleted file mode 100644 index e0b2afe4e1f..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.StupidSingleThreadedHttpServer; -import com.yahoo.search.federation.ProviderConfig.PingOption; -import com.yahoo.search.federation.http.Connection; -import com.yahoo.search.federation.http.HTTPProviderSearcher; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.entity.StringEntity; -import org.junit.Test; - -import java.io.InputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.core.StringContains.containsString; -import static org.junit.Assert.assertThat; - -/** - * See bug #3234696. - * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -public class HttpPostTestCase { - - @Test - public void testPostingSearcher() throws Exception { - StupidSingleThreadedHttpServer server = new StupidSingleThreadedHttpServer(); - server.start(); - - TestPostSearcher searcher = new TestPostSearcher(new ComponentId("foo:1"), - Arrays.asList(new Connection("localhost", server.getServerPort())), - "/"); - Query q = new Query(""); - q.setTimeout(10000000L); - Execution e = new Execution(searcher, Execution.Context.createContextStub()); - - searcher.search(q, e); - - assertThat(server.getRequest(), containsString("My POST body")); - server.stop(); - } - - private static class TestPostSearcher extends HTTPProviderSearcher { - public TestPostSearcher(ComponentId id, List<Connection> connections, String path) { - super(id, connections, httpParameters(path), Statistics.nullImplementation); - } - - private static HTTPParameters httpParameters(String path) { - HTTPParameters httpParameters = new HTTPParameters(path); - httpParameters.setPingOption(PingOption.Enum.DISABLE); - return httpParameters; - } - - @Override - protected HttpUriRequest createRequest(String method, URI uri, HttpEntity entity) { - HttpPost request = new HttpPost(uri); - request.setEntity(entity); - return request; - } - - @Override - protected HttpEntity getRequestEntity(Query query, Hit requestMeta) { - try { - return new StringEntity("My POST body"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return new HashMap<>(0); - } - - @Override - public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException { - // do nothing with the result - } - - @Override - protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { - //Empty - } - } -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java deleted file mode 100644 index b4f3c13b8e2..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.StupidSingleThreadedHttpServer; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import javax.xml.bind.JAXBException; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * Rudimentary http searcher test. - * - * @author bratseth - */ -public class HttpTestCase { - - private StupidSingleThreadedHttpServer httpServer; - private TestHTTPClientSearcher searcher; - - @Test - public void testSearcher() throws JAXBException { - Result result = searchUsingLocalhost(); - - assertEquals("ok", result.getQuery().properties().get("gotResponse")); - assertEquals(0, result.getQuery().errors().size()); - } - - private Result searchUsingLocalhost() { - searcher = new TestHTTPClientSearcher("test","localhost",getPort()); - Query query = new Query("/?query=test"); - - query.setWindow(0,10); - return searcher.search(query, new Execution(searcher, Execution.Context.createContextStub())); - } - - @Test - public void test_that_ip_address_set_on_meta_hit() { - Result result = searchUsingLocalhost(); - Hit metaHit = getFirstMetaHit(result.hits()); - String ip = (String) metaHit.getField(HTTPSearcher.LOG_IP_ADDRESS); - - assertEquals(ip, "127.0.0.1"); - } - - private Hit getFirstMetaHit(HitGroup hits) { - for (Iterator<Hit> i = hits.unorderedDeepIterator(); i.hasNext();) { - Hit hit = i.next(); - if (hit.isMeta()) - return hit; - } - return null; - } - - @Before - public void setUp() throws Exception { - httpServer = new StupidSingleThreadedHttpServer(0, 0) { - @Override - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 5\r\n" + - "\r\n" + - "hello"); - } - }; - httpServer.start(); - } - - private int getPort() { - return httpServer.getServerPort(); - } - - @After - public void tearDown() throws Exception { - httpServer.stop(); - if (searcher != null) { - searcher.shutdownConnectionManagers(); - } - } - - private static class TestHTTPClientSearcher extends HTTPClientSearcher { - - public TestHTTPClientSearcher(String id, String hostName, int port) { - super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); - } - - private static List<Connection> toConnections(String hostName,int port) { - List<Connection> connections=new ArrayList<>(); - connections.add(new Connection(hostName,port)); - return connections; - } - - @Override - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - query.properties().set("gotResponse","ok"); - return query; - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return null; - } - - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java deleted file mode 100644 index 8c1ff69666b..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.prelude.Ping; -import com.yahoo.prelude.Pong; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.StupidSingleThreadedHttpServer; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import com.yahoo.text.Utf8; -import com.yahoo.yolean.Exceptions; -import org.apache.http.HttpEntity; -import org.junit.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Check for different keep-alive scenarios. What we really want to test - * is the server does not hang. - * - * @author Steinar Knutsen - */ -public class PingTestCase { - - private static final int TIMEOUT_MS = 60000; - - @Test - public void testNiceCase() throws Exception { - NiceStupidServer server = new NiceStupidServer(); - server.start(); - checkSearchAndPing(true, true, true, server.getServerPort()); - server.stop(); - } - - private void checkSearchAndPing(boolean firstSearch, boolean pongCheck, boolean secondSearch, int port) { - String resultThing; - String comment; - TestHTTPClientSearcher searcher = new TestHTTPClientSearcher("test", - "localhost", port); - try { - - Query query = new Query("/?query=test"); - - query.setWindow(0, 10); - // high timeout to allow for overloaded test machine - query.setTimeout(TIMEOUT_MS); - Ping ping = new Ping(TIMEOUT_MS); - - long start = System.currentTimeMillis(); - Execution exe = new Execution(searcher, Execution.Context.createContextStub()); - exe.search(query); - - resultThing = firstSearch ? "ok" : null; - comment = firstSearch ? "First search should have succeeded." : "First search should fail."; - assertEquals(comment, resultThing, query.properties().get("gotResponse")); - Pong pong = searcher.ping(ping, searcher.getConnection()); - if (pongCheck) { - assertEquals("Ping should not have failed.", 0, pong.getErrorSize()); - } else { - assertEquals("Ping should have failed.", 1, pong.getErrorSize()); - } - exe = new Execution(searcher, Execution.Context.createContextStub()); - exe.search(query); - - resultThing = secondSearch ? "ok" : null; - comment = secondSearch ? "Second search should have succeeded." : "Second search should fail."; - - assertEquals(resultThing, query.properties().get("gotResponse")); - long duration = System.currentTimeMillis() - start; - // target for duration based on the timeout values + some slack - assertTrue("This test probably hanged.", duration < TIMEOUT_MS + 4000); - searcher.shutdownConnectionManagers(); - } finally { - searcher.deconstruct(); - } - } - - @Test - public void testUselessCase() throws Exception { - UselessStupidServer server = new UselessStupidServer(); - server.start(); - checkSearchAndPing(false, true, false, server.getServerPort()); - server.stop(); - } - - @Test - public void testGrumpyCase() throws Exception { - GrumpyStupidServer server = new GrumpyStupidServer(); - server.start(); - checkSearchAndPing(false, false, false, server.getServerPort()); - server.stop(); - } - - @Test - public void testPassiveAggressiveCase() throws Exception { - PassiveAggressiveStupidServer server = new PassiveAggressiveStupidServer(); - server.start(); - checkSearchAndPing(true, false, true, server.getServerPort()); - server.stop(); - } - - // OK on ping and search - private static class NiceStupidServer extends StupidSingleThreadedHttpServer { - private NiceStupidServer() throws IOException { - super(0, 0); - } - - @Override - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 6\r\n" + - "\r\n" + - "hello\n"); - } - } - - // rejects ping and accepts search - private static class PassiveAggressiveStupidServer extends StupidSingleThreadedHttpServer { - - private PassiveAggressiveStupidServer() throws IOException { - super(0, 0); - } - - @Override - protected byte[] getResponse(String request) { - if (request.contains("/ping")) { - return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 8\r\n" + - "\r\n" + - "go away\n"); - } else { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 6\r\n" + - "\r\n" + - "hello\n"); - } - } - } - - // accepts ping and rejects search - private static class UselessStupidServer extends StupidSingleThreadedHttpServer { - private UselessStupidServer() throws IOException { - super(0, 0); - } - - - @Override - protected byte[] getResponse(String request) { - if (request.contains("/ping")) { - return Utf8.toBytes("HTTP/1.1 200 OK\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 6\r\n" + - "\r\n" + - "hello\n"); - } else { - return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 8\r\n" + - "\r\n" + - "go away\n"); - } - } - } - - // rejects ping and search - private static class GrumpyStupidServer extends StupidSingleThreadedHttpServer { - private GrumpyStupidServer() throws IOException { - super(0, 0); - } - - @Override - protected byte[] getResponse(String request) { - return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" + - "Content-Type: text/xml; charset=UTF-8\r\n" + - "Connection: close\r\n" + - "Content-Length: 8\r\n" + - "\r\n" + - "go away\n"); - } - } - - private static class TestHTTPClientSearcher extends HTTPClientSearcher { - - public TestHTTPClientSearcher(String id, String hostName, int port) { - super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation); - } - - private static List<Connection> toConnections(String hostName,int port) { - List<Connection> connections=new ArrayList<>(); - connections.add(new Connection(hostName,port)); - return connections; - } - - @Override - public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException { - query.properties().set("gotResponse","ok"); - return query; - } - - @Override - public Result search(Query query, Execution execution, - Connection connection) { - URI uri; - try { - uri = new URL("http", connection.getHost(), connection - .getPort(), "/search").toURI(); - } catch (MalformedURLException e) { - query.errors().add(createMalformedUrlError(query, e)); - return execution.search(query); - } catch (URISyntaxException e) { - query.errors().add(createMalformedUrlError(query, e)); - return execution.search(query); - } - - HttpEntity entity; - try { - entity = getEntity(uri, query); - } catch (IOException e) { - query.errors().add( - ErrorMessage.createBackendCommunicationError("Error when trying to connect to HTTP backend in " - + this + " using " + connection - + " for " + query + ": " - + Exceptions.toMessageString(e))); - return execution.search(query); - } catch (TimeoutException e) { - query.errors().add(ErrorMessage.createTimeout("No time left for HTTP traffic in " - + this - + " for " + query + ": " + e.getMessage())); - return execution.search(query); - } - if (entity == null) { - query.errors().add( - ErrorMessage.createBackendCommunicationError("No result from connecting to HTTP backend in " - + this + " using " + connection + " for " + query)); - return execution.search(query); - } - - try { - query = handleResponse(entity, query); - } catch (IOException e) { - query.errors().add( - ErrorMessage.createBackendCommunicationError("Error when trying to consume input in " - + this + ": " + Exceptions.toMessageString(e))); - } finally { - cleanupHttpEntity(entity); - } - return execution.search(query); - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return null; - } - - @Override - protected URI getPingURI(Connection connection) - throws MalformedURLException, URISyntaxException { - return new URL("http", connection.getHost(), connection.getPort(), "/ping").toURI(); - } - - Connection getConnection() { - return getHasher().getNodes().select(0, 0); - } - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java deleted file mode 100644 index 19750cf84cc..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.http; - -import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.statistics.Statistics; -import com.yahoo.vespa.defaults.Defaults; -import org.junit.Test; - -import java.util.Collections; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * Tests that source and backend specific parameters from the query are added correctly to the backend requests - * - * @author bratseth - */ -public class QueryParametersTestCase { - - @Test - public void testQueryParameters() { - Query query=new Query(); - query.properties().set("a","a-value"); - query.properties().set("b.c","b.c-value"); - query.properties().set("source.otherSource.d","d-value"); - query.properties().set("source.testSource.e","e-value"); - query.properties().set("source.testSource.f.g","f.g-value"); - query.properties().set("provider.testProvider.h","h-value"); - query.properties().set("provider.testProvider.i.j","i.j-value"); - - query.properties().set("sourceName","testSource"); // Done by federation searcher - query.properties().set("providerName","testProvider"); // Done by federation searcher - - TestHttpProvider searcher=new TestHttpProvider(); - Map<String,String> parameters=searcher.getQueryMap(query); - searcher.deconstruct(); - - assertEquals(4,parameters.size()); // the appropriate 4 of the above - assertEquals(parameters.get("e"),"e-value"); - assertEquals(parameters.get("f.g"),"f.g-value"); - assertEquals(parameters.get("h"),"h-value"); - assertEquals(parameters.get("i.j"),"i.j-value"); - } - - public static class TestHttpProvider extends HTTPProviderSearcher { - - public TestHttpProvider() { - super(new ComponentId("test"), Collections.singletonList(new Connection("host", Defaults.getDefaults().vespaWebServicePort())), "path", Statistics.nullImplementation); - } - - @Override - public Map<String, String> getCacheKey(Query q) { - return Collections.singletonMap("nocaching", String.valueOf(Math.random())); - } - - @Override - protected void fill(Result result, String summaryClass, Execution execution, Connection connection) { - } - - } - -} - diff --git a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore +++ /dev/null diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java deleted file mode 100644 index 022177bc42b..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.language.Linguistics; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; -import com.yahoo.search.Query; -import com.yahoo.search.federation.vespa.QueryMarshaller; -import com.yahoo.search.searchchain.Execution; -import com.yahoo.search.test.QueryTestCase; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class QueryMarshallerTestCase { - - private static final Linguistics linguistics = new SimpleLinguistics(); - - @Test - public void testCommonCommonCase() { - AndItem root = new AndItem(); - addThreeWords(root); - assertEquals("a AND b AND c", new QueryMarshaller().marshal(root)); - } - - @Test - public void testPhrase() { - PhraseItem root = new PhraseItem(); - root.setIndexName("habla"); - addThreeWords(root); - assertEquals("habla:\"a b c\"", new QueryMarshaller().marshal(root)); - } - - @Test - public void testPhraseDefaultIndex() { - PhraseItem root = new PhraseItem(); - addThreeWords(root); - assertEquals("\"a b c\"", new QueryMarshaller().marshal(root)); - } - - @Test - public void testLittleMoreComplex() { - AndItem root = new AndItem(); - addThreeWords(root); - OrItem ambig = new OrItem(); - root.addItem(ambig); - addThreeWords(ambig); - AndItem but = new AndItem(); - addThreeWords(but); - ambig.addItem(but); - assertEquals("a AND b AND c AND ( a OR b OR c OR ( a AND b AND c ) )", - new QueryMarshaller().marshal(root)); - } - - @Test - public void testRank() { - RankItem root = new RankItem(); - addThreeWords(root); - assertEquals("a RANK b RANK c", new QueryMarshaller().marshal(root)); - } - - @Test - public void testNear() { - NearItem near = new NearItem(3); - addThreeWords(near); - assertEquals("a NEAR(3) b NEAR(3) c", new QueryMarshaller().marshal(near)); - } - - @Test - public void testONear() { - ONearItem oNear = new ONearItem(3); - addThreeWords(oNear); - assertEquals("a ONEAR(3) b ONEAR(3) c", new QueryMarshaller().marshal(oNear)); - } - - private void addThreeWords(CompositeItem root) { - root.addItem(new WordItem("a")); - root.addItem(new WordItem("b")); - root.addItem(new WordItem("c")); - } - - @Test - public void testNegativeGroupedTerms() { - testQueryString(new QueryMarshaller(), "a -(b c) -(d e)", - "a ANDNOT ( b AND c ) ANDNOT ( d AND e )"); - } - - @Test - public void testPositiveGroupedTerms() { - testQueryString(new QueryMarshaller(), "a (b c)", "a AND ( b OR c )"); - } - - @Test - public void testInt() { - testQueryString(new QueryMarshaller(), "yahoo 123", "yahoo AND 123"); - } - - @Test - public void testCJKOneWord() { - testQueryString(new QueryMarshaller(), "天龍人"); - } - - @Test - public void testTwoWords() { - testQueryString(new QueryMarshaller(), "John Smith", "John AND Smith", null, new SimpleLinguistics()); - } - - @Test - public void testTwoWordsInPhrase() { - testQueryString(new QueryMarshaller(), "\"John Smith\"", "\"John Smith\"", null, new SimpleLinguistics()); - } - - @Test - public void testCJKTwoSentences() { - testQueryString(new QueryMarshaller(), "是不是這樣的夜晚 你才會這樣地想起我", "是不是這樣的夜晚 AND 你才會這樣地想起我"); - } - - @Test - public void testCJKTwoSentencesWithLanguage() { - testQueryString(new QueryMarshaller(), "助妳好孕 生1胎北市發2萬", "助妳好孕 AND 生1胎北市發2萬", "zh-Hant"); - } - - @Test - public void testCJKTwoSentencesInPhrase() { - QueryMarshaller marshaller = new QueryMarshaller(); - testQueryString(marshaller, "\"助妳好孕 生1胎北市發2萬\"", "\"助妳好孕 生1胎北市發2萬\"", "zh-Hant"); - testQueryString(marshaller, "\"是不是這樣的夜晚 你才會這樣地想起我\"", "\"是不是這樣的夜晚 你才會這樣地想起我\""); - } - - @Test - public void testCJKMultipleSentences() { - testQueryString(new QueryMarshaller(), "염부장님과 함께했던 좋은 추억들은", "염부장님과 AND 함께했던 AND 좋은 AND 추억들은"); - } - - @Test - public void testIndexRestriction() { - /** ticket 3707606, comment #29 */ - testQueryString(new QueryMarshaller(), "site:nytimes.com", "site:\"nytimes com\""); - } - - private void testQueryString(QueryMarshaller marshaller, String uq) { - testQueryString(marshaller, uq, uq, null); - } - - private void testQueryString(QueryMarshaller marshaller, String uq, String mq) { - testQueryString(marshaller, uq, mq, null); - } - - private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang) { - testQueryString(marshaller, uq, mq, lang, linguistics); - } - - private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang, Linguistics linguistics) { - Query query = new Query("/?query=" + QueryTestCase.httpEncode(uq) + ((lang != null) ? "&language=" + lang : "")); - query.getModel().setExecution(new Execution(new Execution.Context(null, new IndexFacts(), null, null, linguistics))); - assertEquals(mq, marshaller.marshal(query.getModel().getQueryTree().getRoot())); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java deleted file mode 100644 index 8d1ecf4085e..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.search.Query; -import com.yahoo.search.federation.vespa.VespaSearcher; -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * Tests that source and backend specific parameters from the query are added correctly to the backend requests - * - * @author bratseth - */ -public class QueryParametersTestCase { - - public void testQueryParameters() { - Query query=new Query(); - query.properties().set("a","a-value"); - query.properties().set("b.c","b.c-value"); - query.properties().set("source.otherSource.d","d-value"); - query.properties().set("source.testSource.e","e-value"); - query.properties().set("source.testSource.f.g","f.g-value"); - query.properties().set("provider.testProvider.h","h-value"); - query.properties().set("provider.testProvider.i.j","i.j-value"); - - query.properties().set("sourceName","testSource"); // Done by federation searcher - query.properties().set("providerName","testProvider"); // Done by federation searcher - - VespaSearcher searcher=new VespaSearcher("testProvider","",0,""); - Map<String,String> parameters=searcher.getQueryMap(query); - searcher.deconstruct(); - - assertEquals(9, parameters.size()); // 5 standard + the appropriate 4 of the above - assertEquals(parameters.get("e"),"e-value"); - assertEquals(parameters.get("f.g"),"f.g-value"); - assertEquals(parameters.get("h"),"h-value"); - assertEquals(parameters.get("i.j"),"i.j-value"); - } - -} - diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java deleted file mode 100644 index 50186e03ae8..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import java.util.Iterator; - -import com.yahoo.net.URI; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.federation.vespa.ResultBuilder; -import com.yahoo.search.result.ErrorHit; -import com.yahoo.search.result.ErrorMessage; -import com.yahoo.search.result.HitGroup; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -/** - * Test XML parsing of results. - * - * @author Steinar Knutsen - */ -@SuppressWarnings("deprecation") -public class ResultBuilderTestCase { - - private boolean quickCompare(double a, double b) { - double z = Math.min(Math.abs(a), Math.abs(b)); - if (Math.abs((a - b)) < (z / 1e14)) { - return true; - } else { - return false; - } - } - - @Test - public void testSimpleResult() { - boolean gotErrorDetails = false; - ResultBuilder r = new ResultBuilder(); - Result res = r.parse("file:src/test/java/com/yahoo/prelude/searcher/test/testhit.xml", new Query("?query=a")); - assertEquals(3, res.getConcreteHitCount()); - assertEquals(4, res.getHitCount()); - ErrorHit e = (ErrorHit) res.hits().get(0); - // known problem, if the same error is the main error is - // in details, it'll be added twice. Not sure how to fix that, - // because old Vespa systems give no error details, and there - // is no way of nuking an existing error if the details exist. - for (Iterator<?> i = e.errorIterator(); i.hasNext();) { - ErrorMessage err = (ErrorMessage) i.next(); - assertEquals(5, err.getCode()); - String details = err.getDetailedMessage(); - if (details != null) { - gotErrorDetails = true; - assertEquals("An error as ordered", details.trim()); - } - } - assertTrue("Error details are missing", gotErrorDetails); - assertEquals(new URI("http://def"), res.hits().get(1).getId()); - assertEquals("test/stuff\\tsome/other", res.hits().get(2).getField("category")); - assertEquals("<field>habla</field>" - + "<hi>blbl</hi><br /><>&fdlkkgj</field>;lk<a b=\"1\" c=\"2\" />" - + "<x><y><z /></y></x>", res.hits().get(3).getField("annoying").toString()); - } - - @Test - public void testNestedResult() { - ResultBuilder r = new ResultBuilder(); - Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml", new Query("?query=a")); - assertNull(res.hits().getError()); - assertEquals(3, res.hits().size()); - assertEquals("ABCDEFGHIJKLMNOPQRSTUVWXYZ", res.hits().get(0).getField("guid").toString()); - HitGroup g1 = (HitGroup) res.hits().get(1); - HitGroup g2 = (HitGroup) res.hits().get(2); - assertEquals(15, g1.size()); - assertEquals("reward_for_thumb", g1.get(1).getField("id").toString()); - assertEquals(10, g2.size()); - HitGroup g3 = (HitGroup) g2.get(3); - assertEquals("badge", g3.types().iterator().next()); - assertEquals(2, g3.size()); - assertEquals("badge/Topic Explorer 5", g3.get(0).getField("name").toString()); - } - - @Test - public void testWeirdDocumentID() { - ResultBuilder r = new ResultBuilder(); - Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml", new Query("?query=a")); - assertNull(res.hits().getError()); - assertEquals(3, res.hits().size()); - assertEquals(new URI("nalle"), res.hits().get(0).getId()); - assertEquals(new URI("tralle"), res.hits().get(1).getId()); - assertEquals(new URI("kalle"), res.hits().get(2).getId()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java deleted file mode 100644 index 7fbe883bfb7..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.component.chain.Chain; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.Searcher; -import com.yahoo.search.federation.vespa.VespaSearcher; -import com.yahoo.search.searchchain.Execution; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class VespaIntegrationTestCase { - - // TODO: Setup the answering vespa searcher from this test.... - @Test - public void testIt() { - if (System.currentTimeMillis() > 0) return; - Chain<Searcher> chain = new Chain<>(new VespaSearcher("test","example.yahoo.com",19010,"")); - Result result = new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=test")); - assertEquals(23, result.hits().size()); - } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java deleted file mode 100644 index cc00c6739fa..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.federation.vespa.test; - -import com.yahoo.prelude.query.*; -import com.yahoo.search.Query; -import com.yahoo.search.federation.vespa.VespaSearcher; -import com.yahoo.search.query.QueryTree; -import com.yahoo.search.query.parser.Parsable; -import com.yahoo.search.query.parser.Parser; -import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; -import com.yahoo.search.result.Hit; -import com.yahoo.search.searchchain.Execution; -import org.apache.http.HttpEntity; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; -import java.net.URI; - -import static org.junit.Assert.assertEquals; - -/** - * Check query marshaling in VespaSearcher works. - * - * @author Steinar Knutsen - */ -public class VespaSearcherTestCase { - - private VespaSearcher searcher; - - @Before - public void setUp() { - searcher = new VespaSearcher("cache1","",0,""); - } - - @After - public void tearDown() { - searcher.deconstruct(); - } - - @Test - public void testMarshalQuery() { - RankItem root = new RankItem(); - QueryTree r = new QueryTree(root); - AndItem recall = new AndItem(); - PhraseItem usual = new PhraseItem(); - PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); - WordItem filterWord = new WordItem("silly"); - - filterPhrase.setFilter(true); - filterWord.setFilter(true); - - root.addItem(recall); - usual.addItem(new WordItem("new")); - usual.addItem(new WordItem("york")); - recall.addItem(usual); - recall.addItem(new WordItem("shoes")); - root.addItem(new WordItem("nike")); - root.addItem(new WordItem("adidas")); - root.addItem(filterPhrase); - recall.addItem(filterWord); - - assertEquals("( \"new york\" AND shoes AND silly ) RANK nike RANK adidas RANK \"bloody expensive\"", searcher.marshalQuery(r)); - } - - @Test - public void testMarshalQuerySmallTree() { - RankItem root = new RankItem(); - QueryTree r = new QueryTree(root); - AndItem recall = new AndItem(); - PhraseItem usual = new PhraseItem(); - PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"}); - WordItem filterWord = new WordItem("silly"); - - filterPhrase.setFilter(true); - filterWord.setFilter(true); - - root.addItem(recall); - usual.addItem(new WordItem("new")); - usual.addItem(new WordItem("york")); - recall.addItem(usual); - recall.addItem(new WordItem("shoes")); - root.addItem(filterPhrase); - recall.addItem(filterWord); - - assertEquals("( \"new york\" AND shoes AND silly ) RANK \"bloody expensive\"", searcher.marshalQuery(r)); - // TODO: Switch to this 2-way check rather than just 1-way and then also make this actually treat filter terms correctly - // assertMarshals(root) - } - - @Test - public void testWandMarshalling() { - WeakAndItem root = new WeakAndItem(); - root.setN(32); - root.addItem(new WordItem("a")); - root.addItem(new WordItem("b")); - root.addItem(new WordItem("c")); - assertMarshals(root); - } - - @Test - public void testWandMarshalling2() { - // AND (WAND(10) a!1 the!10) source:yahoonews - AndItem root = new AndItem(); - WeakAndItem wand = new WeakAndItem(10); - wand.addItem(newWeightedWordItem("a",1)); - wand.addItem(newWeightedWordItem("the",10)); - root.addItem(wand); - root.addItem(new WordItem("yahoonews","source")); - assertMarshals(root); - } - - private WordItem newWeightedWordItem(String word,int weight) { - WordItem wordItem=new WordItem(word); - wordItem.setWeight(weight); - return wordItem; - } - - private void assertMarshals(Item root) { - QueryTree r = new QueryTree(root); - String marshalledQuery=searcher.marshalQuery(r); - assertEquals("Marshalled form '" + marshalledQuery + "' recreates the original", - r,parseQuery(marshalledQuery,"")); - } - - private static Item parseQuery(String query, String filter) { - Parser parser = ParserFactory.newInstance(Query.Type.ADVANCED, new ParserEnvironment()); - return parser.parse(new Parsable().setQuery(query).setFilter(filter)); - } - - @Test - public void testSourceProviderProperties() throws Exception { - /* TODO: update test - Server httpServer = new Server(); - try { - SocketConnector listener = new SocketConnector(); - listener.setHost("0.0.0.0"); - httpServer.addConnector(listener); - httpServer.setHandler(new DummyHandler()); - httpServer.start(); - - int port=httpServer.getConnectors()[0].getLocalPort(); - - List<SourcesConfig.Source> sourcesConfig = new ArrayList<SourcesConfig.Source>(); - SourcesConfig.Source sourceConfig = new SourcesConfig.Source(); - sourceConfig.chain.setValue("news"); - sourceConfig.provider.setValue("news"); - sourceConfig.id.setValue("news"); - sourceConfig.timelimit.value = 10000; - sourcesConfig.add(sourceConfig); - FederationSearcher federator = - new FederationSearcher(ComponentId.createAnonymousComponentId(), - new ArrayList<SourcesConfig.Source>(sourcesConfig)); - SearchChain mainChain=new OrderedSearchChain(federator); - - SearchChainRegistry registry=new SearchChainRegistry(); - SearchChain sourceChain=new SearchChain(new ComponentId("news"),new VespaSearcher("test","localhost",port,"")); - registry.register(sourceChain); - Query query=new Query("?query=hans&hits=20&provider.news.a=a1&source.news.b=b1"); - Result result=new Execution(mainChain,registry).search(query); - assertNull(result.hits().getError()); - Hit testHit=result.hits().get("testHit"); - assertNotNull(testHit); - assertEquals("testValue",testHit.fields().get("testField")); - assertEquals("a1",testHit.fields().get("a")); - assertEquals("b1",testHit.fields().get("b")); - } - finally { - httpServer.stop(); - } - */ - } - - @Test - public void testVespaSearcher() { - VespaSearcher v=new VespaSearcherValidatingSubclass(); - new Execution(v, Execution.Context.createContextStub()).search(new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&filter=myfilter"))); - } - - private class VespaSearcherValidatingSubclass extends VespaSearcher { - - public VespaSearcherValidatingSubclass() { - super("configId","host",80,"path"); - } - - @Override - protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException { - assertEquals("http://host:80/path?query=test+RANK+myfilter&type=adv&offset=0&hits=10&presentation.format=xml",uri.toString()); - return super.getEntity(uri,requestMeta,query); - } - - } - - // used by the old testSourceProviderProperties() -// private class DummyHandler extends AbstractHandler { -// public void handle(String s, Request request, HttpServletRequest httpServletRequest, -// HttpServletResponse httpServletResponse) throws IOException, ServletException { -// -// try { -// Response httpResponse = httpServletResponse instanceof Response ? (Response) httpServletResponse : HttpConnection.getCurrentConnection().getResponse(); -// -// httpResponse.setStatus(HttpStatus.OK_200); -// httpResponse.setContentType("text/xml"); -// httpResponse.setCharacterEncoding("UTF-8"); -// Result r=new Result(new Query()); -// Hit testHit=new Hit("testHit"); -// testHit.setField("uri","testHit"); // That this is necessary is quite unfortunate... -// testHit.setField("testField","testValue"); -// // Write back all incoming properties: -// for (Object e : httpServletRequest.getParameterMap().entrySet()) { -// Map.Entry entry=(Map.Entry)e; -// testHit.setField(entry.getKey().toString(),getFirstValue(entry.getValue())); -// } -// -// r.hits().add(testHit); -// -// //StringWriter sw=new StringWriter(); -// //r.render(sw); -// //System.out.println(sw.toString()); -// -// SearchRendererAdaptor.callRender(httpResponse.getWriter(), r); -// httpResponse.complete(); -// } -// catch (Exception e) { -// System.out.println("WARNING: Could not respond to request: " + Exceptions.toMessageString(e)); -// e.printStackTrace(); -// } -// } -// -// private String getFirstValue(Object entry) { -// if (entry instanceof String[]) -// return ((String[])entry)[0].toString(); -// else -// return entry.toString(); -// } -// } - -} diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml deleted file mode 100644 index c17bbf8474d..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result total-hit-count="3"> - <hit relevancy="75" source="test" type="summary"> - <field name="uri">nalle</field> - <field name="relevancy">75</field> - <field name="collapseId">0</field> - </hit> - <hit relevancy="73" source="test" type="summary test other"> - <field name="documentId">tralle</field> - <field name="relevancy">73</field> - <field name="collapseId">0</field> - <field name="category">test/stuff\tsome/other</field> - <field name="bsumtitle">dklf øæå sdf > & < -Ipsum, etc.</field> - </hit> - <hit relevancy="70" source="test" type="summary"> - <field name="DOCUMENTID">kalle</field> - <field name="relevancy">75</field> - <field name="collapseId">0</field> - <field name="annoying"><field>habla</field><hi>blbl</hi><br /><![CDATA[<>&fdlkkgj</field>]]>;lk<a b="1" c="2" /><x><y><z /></y></x></field> - </hit> -</result> diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml deleted file mode 100644 index 8b5ab710378..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml +++ /dev/null @@ -1,318 +0,0 @@ -<?xml version="1.0" encoding="utf-8" ?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<result total-hit-count="36"> -<hit type="user_reputation"> -<field name="guid">ABCDEFGHIJKLMNOPQRSTUVWXYZ</field> -<field name="level">zero</field> -<field name="points">0</field> -<field name="created">1287600988</field> -<field name="updated">1287600988</field> -</hit> -<group type="actions"> -<hit type="action"> -<field name="id">thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> -<hit type="action"> -<field name="id">reward_for_thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> -<hit type="action"> -<field name="id">undo_thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">buzz</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">undo_reward_for_thumb</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">vote</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">report_abuse</field> -<field name="created">1287600992</field> -<field name="updated">1287600992</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">reward_for_vote</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">signup</field> -<field name="created">1287600993</field> -<field name="updated">1287600993</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">registered</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">get_points</field> -<field name="created">1287600989</field> -<field name="updated">1287600989</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">contrib_SignedUp</field> -<field name="created">1287600993</field> -<field name="updated">1287600993</field> -<field name="points">0</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">contrib_AgreedToTos</field> -<field name="created">1287600993</field> -<field name="updated">1287600993</field> -<field name="points">500</field> -<field name="level">zero</field> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">Create Feature</field> -<field name="created"/> -<field name="updated"/> -<field name="points">0</field> -<field name="level"/> -<field name="isEnabled">1</field> -</hit> - -<hit type="action"> -<field name="id">add_theme</field> -<field name="created"/> -<field name="updated"/> -<field name="points">0</field> -<field name="level"/> -<field name="isEnabled">1</field> -</hit> -</group> - -<group type="awards"> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/First Feature</field> -<field name="description">You’ve created your First Feature!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/1stfeature.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/25th Feature</field> -<field name="description">You’ve created your 25th Feature!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/25thfeature.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/50th Feature</field> -<field name="description">You’ve created your 50th Feature!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/10thfeature.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Topic Explorer 5</field> -<field name="description">You’ve added a Feature to your 5th Topic Page!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/5thtopic.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Topic Explorer 15</field> -<field name="description">You’ve added a Feature to your 15th Topic Page!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/15thtopic.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Topic Explorer 30</field> -<field name="description">You’ve added a Feature to your 30th Topic Page!</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/30thtopic.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> - -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> - -<group type="badge"> - -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Pollster</field> -<field name="description">You’ve created your 5th Poll Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/pollster.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -<group type="badge"> -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Reporter</field> -<field name="description">You’ve created your 5th Article Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/newsreporter.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -<group type="badge"> -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Paparazzi</field> -<field name="description">You’ve created your 5th Image Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/paparazzi.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -<group type="badge"> -<hit type="info"> -<field name="type">badge</field> -<field name="name">badge/Video Reporter</field> -<field name="description">You’ve created your 5th Video Feature.</field> -<field name="status">active</field> -<field name="imageUrl">http://example.yahoo.com/director.png</field> -<field name="imageHeight">57</field> -<field name="imageWidth">57</field> -</hit> -<hit type="earned"> -<field name="date">1283981088</field> -<field name="context">topic/Jennifer_Aniston</field> -</hit> -</group> -</group> -</result> diff --git a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore +++ /dev/null diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java index 999a6d32ac6..569f219f635 100644 --- a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java @@ -3,9 +3,6 @@ package com.yahoo.search.pagetemplates.engine.test; import com.google.common.util.concurrent.ListenableFuture; import com.yahoo.io.IOUtils; -import com.yahoo.prelude.templates.TiledTemplateSet; -import com.yahoo.prelude.templates.UserTemplate; -import com.yahoo.prelude.templates.test.TilingTestCase; import com.yahoo.search.Result; import com.yahoo.search.pagetemplates.PageTemplate; import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; diff --git a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java deleted file mode 100644 index 11d163373ac..00000000000 --- a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.result.test; - -import java.io.IOException; - -import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.prelude.templates.test.TilingTestCase; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author bratseth - */ -public class ArrayOutputTestCase { - - @Test - public void testArrayOutput() throws IOException { - Result r=new Result(new Query("?query=ignored")); - Hit hit=new Hit("test"); - hit.setField("phone",new XMLString("\n <item>408-555-1234</item>" + "\n <item>408-555-5678</item>\n ")); - r.hits().add(hit); - - String rendered = TilingTestCase.getRendered(r); - String[] lines= rendered.split("\n"); - assertEquals(" <field name=\"phone\">",lines[4]); - assertEquals(" <item>408-555-1234</item>",lines[5]); - assertEquals(" <item>408-555-5678</item>",lines[6]); - assertEquals(" </field>",lines[7]); - } - -} |