aboutsummaryrefslogtreecommitdiffstats
path: root/container-search
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@oath.com>2018-11-05 13:33:19 +0100
committergjoranv <gv@oath.com>2019-01-21 15:09:25 +0100
commit8505bd016ec028c4dcdeb04ec4baa07a435e166b (patch)
treee63e493940d11732214b240269f5238cfe7e6903 /container-search
parent6b01125fad374672738a0e68edf7eef5f5463ec9 (diff)
Remove HTTP searchers
Diffstat (limited to 'container-search')
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java40
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java71
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java28
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/Connection.java33
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java128
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java279
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java318
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java261
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java961
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java91
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java114
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java23
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/http/package-info.java5
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java176
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java642
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java259
-rw-r--r--container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java5
-rw-r--r--container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java118
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java309
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java74
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java212
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java238
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java99
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java124
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java288
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java67
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/image/.gitignore0
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java160
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java42
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java94
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java28
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java240
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml23
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml318
-rw-r--r--container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore0
-rw-r--r--container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java3
-rw-r--r--container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java35
37 files changed, 0 insertions, 5906 deletions
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java
deleted file mode 100644
index 1607d108722..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPClientSearcher.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import java.util.Collections;
-
-import com.yahoo.component.ComponentId;
-import com.yahoo.search.federation.ProviderConfig;
-import com.yahoo.search.Result;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.statistics.Statistics;
-
-
-/**
- * Superclass for http client searchers which depends on config. All this is doing is translating
- * the provider and cache configurations to parameters which are passed upwards.
- *
- * @author bratseth
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public abstract class ConfiguredHTTPClientSearcher extends HTTPClientSearcher {
-
- /** Create this from a configuraton */
- public ConfiguredHTTPClientSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) {
- super(id, ConfiguredSearcherHelper.toConnectionList(providerConfig), new HTTPParameters(providerConfig), manager);
- }
-
- /** Create an instance from direct parameters having a single connection. Useful for testing */
- public ConfiguredHTTPClientSearcher(String idString,String host,int port,String path, Statistics manager) {
- super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager);
- }
-
- /** Forwards to the next in chain fill(result,summaryName) */
- @Override
- public void fill(Result result,String summaryName, Execution execution,Connection connection) {
- execution.fill(result,summaryName);
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java
deleted file mode 100644
index 9f99a790df0..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredHTTPProviderSearcher.java
+++ /dev/null
@@ -1,71 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.yahoo.component.ComponentId;
-import com.yahoo.search.federation.ProviderConfig;
-import com.yahoo.search.cache.QrBinaryCacheConfig;
-import com.yahoo.search.cache.QrBinaryCacheRegionConfig;
-import com.yahoo.search.Result;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.statistics.Statistics;
-
-import java.util.Collections;
-
-
-/**
- * Superclass for http provider searchers which depends on config. All this is doing is translating
- * the provider and cache configurations to parameters which are passed upwards.
- *
- * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
- * @author bratseth
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public abstract class ConfiguredHTTPProviderSearcher extends HTTPProviderSearcher {
-
- /** Create this from a configuraton */
- public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig, Statistics manager) {
- super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager);
- }
-
- /** Create this from a configuraton */
- public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig,
- HTTPParameters parameters, Statistics manager) {
- super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager);
- }
-
- /** Create this from a configuraton with a configured cache */
- public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig,
- final QrBinaryCacheConfig cacheConfig,
- final QrBinaryCacheRegionConfig regionConfig, Statistics manager) {
- super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),new HTTPParameters(providerConfig), manager);
- configureCache(cacheConfig,regionConfig);
- }
-
- /** Create this from a configuraton with a configured cache */
- public ConfiguredHTTPProviderSearcher(final ComponentId id, final ProviderConfig providerConfig,
- final QrBinaryCacheConfig cacheConfig,
- final QrBinaryCacheRegionConfig regionConfig, HTTPParameters parameters, Statistics manager) {
- super(id,ConfiguredSearcherHelper.toConnectionList(providerConfig),parameters, manager);
- configureCache(cacheConfig,regionConfig);
- }
-
- /** Create an instance from direct parameters having a single connection. Useful for testing */
- public ConfiguredHTTPProviderSearcher(String idString,String host,int port,String path, Statistics manager) {
- super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),path, manager);
- }
-
- /** Create an instance from direct parameters having a single connection. Useful for testing */
- public ConfiguredHTTPProviderSearcher(String idString,String host,int port,HTTPParameters parameters, Statistics manager) {
- super(new ComponentId(idString), Collections.singletonList(new Connection(host,port)),parameters, manager);
- }
-
- /**
- * Override this to provider multi-phase result filling towards a backend.
- * This default implementation does nothing.
- */
- @Override
- public void fill(Result result,String summaryName, Execution execution,Connection connection) {
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java b/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java
deleted file mode 100644
index 0fa235709f9..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/ConfiguredSearcherHelper.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.yahoo.search.federation.ProviderConfig;
-
-/**
- * Some static helper classes for configured*Searcher classes
- *
- * @author bratseth
- */
-// TODO: Remove on Vespa 7
-class ConfiguredSearcherHelper {
-
- /** No instantiation */
- private ConfiguredSearcherHelper() { }
-
- public static List<Connection> toConnectionList(ProviderConfig providerConfig) {
- List<Connection> connections=new ArrayList<>();
- for(ProviderConfig.Node node : providerConfig.node()) {
- connections.add(new Connection(node.host(), node.port()));
- }
- return connections;
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java b/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java
deleted file mode 100644
index 4f624a12547..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/Connection.java
+++ /dev/null
@@ -1,33 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-/**
- * Represents a connection to a particular node (host/port).
- * Right now this is just a container of connection parameters, but might be extended to
- * contain an open connection later.
- * The host and port state is immutable.
- *
- * @author bratseth
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class Connection {
-
- private String host;
- private int port;
-
- public Connection(String host,int port) {
- this.host=host;
- this.port=port;
- }
-
- public String getHost() { return host; }
-
- public int getPort() { return port; }
-
- public String toString() {
- return "http connection '" + host + ":" + port + "'";
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java
deleted file mode 100644
index 0407ddeca1b..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/GzipDecompressingEntity.java
+++ /dev/null
@@ -1,128 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import org.apache.http.HttpEntity;
-import org.apache.http.entity.HttpEntityWrapper;
-
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.zip.GZIPInputStream;
-
-/**
- * Used by HTTPSearcher when talking to services returning compressed content.
- *
- * @author Mainak Mandal
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class GzipDecompressingEntity extends HttpEntityWrapper {
-
- private static class Resources {
-
- byte [] buffer;
- int total;
-
- Resources() {
- total = 0;
- buffer = new byte[65536];
- }
- void drain(InputStream zipStream) throws IOException {
- int numRead = zipStream.read(buffer, total, buffer.length);
- while (numRead != -1) {
- total += numRead;
- if ((total + 65536) > buffer.length) {
- buffer = Arrays.copyOf(buffer, buffer.length + numRead);
- }
- numRead = zipStream.read(buffer, total, buffer.length - total);
- }
- }
-
- }
-
- private final Resources resources = new Resources();
-
- public GzipDecompressingEntity(final HttpEntity entity) throws IllegalStateException, IOException {
- super(entity);
- GZIPInputStream gz = new GZIPInputStream(entity.getContent());
- InputStream zipStream = new BufferedInputStream(gz);
- try {
- resources.drain(zipStream);
- } catch (IOException e) {
- throw e;
- } finally {
- zipStream.close();
- }
- }
-
- @Override
- public InputStream getContent() throws IOException, IllegalStateException {
-
- final ByteBuffer buff = ByteBuffer.wrap(resources.buffer, 0, resources.total);
- return new InputStream() {
-
- @Override
- public int available() throws IOException {
- return buff.remaining();
- }
-
- @Override
- public int read() throws IOException {
- if (buff.hasRemaining())
- return buff.get() & 0xFF;
-
- return -1;
- }
-
- @Override
- public int read(byte[] b) throws IOException {
- if (!buff.hasRemaining())
- return -1;
-
- int len = b.length;
- if (len > buff.remaining())
- len = buff.remaining();
- buff.get(b, 0, len);
- return len;
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- if (!buff.hasRemaining())
- return -1;
-
- if (len > buff.remaining())
- len = buff.remaining();
- buff.get(b, off, len);
- return len;
- }
-
- @Override
- public long skip(long n) throws IOException {
- if (!buff.hasRemaining())
- return -1;
-
- if (n > buff.remaining())
- n = buff.remaining();
-
- buff.position(buff.position() + (int) n);
- return n;
- }
- };
- }
-
- @Override
- public long getContentLength() {
- return resources.total;
- }
-
- @Override
- public void writeTo(OutputStream outstream) throws IOException {
- outstream.write(resources.buffer, 0, resources.total);
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java
deleted file mode 100644
index ee093adc6fc..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPClientSearcher.java
+++ /dev/null
@@ -1,279 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.yahoo.component.ComponentId;
-import com.yahoo.jdisc.http.CertificateStore;
-import com.yahoo.yolean.Exceptions;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.processing.request.CompoundName;
-import com.yahoo.search.result.ErrorMessage;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.statistics.Statistics;
-
-import org.apache.http.HttpEntity;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Logger;
-
-/**
- * A utility parent for searchers which gets data from web services which is incorporated into the query.
- * This searcher will take care of implementing the search method while the extending class implements
- * {@link #getQueryMap} and {@link #handleResponse} to create the http request and handle the response, respectively.
- *
- * <p>This class automatically adds a meta hit containing latency and other
- * meta information about the obtained HTTP data using createRequestMeta().
- * The fields available in the hit are:</p>
- *
- * <dl><dt>
- * HTTPSearcher.LOG_LATENCY_START
- * <dd>
- * The latency of the external provider answering a request.
- * <dt>
- * HTTPSearcher.LOG_LATENCY_FINISH
- * <dd>
- * Total time of the HTTP traffic, but also decoding of the data, is this
- * happens at the same time.
- * <dt>
- * HTTPSearcher.LOG_URI
- * <dd>
- * The complete URI used for external service.
- * <dt>
- * HTTPSearcher.LOG_SCHEME
- * <dd>
- * The scheme of the request URI sent.
- * <dt>
- * HTTPSearcher.LOG_HOST
- * <dd>
- * The host used for the request URI sent.
- * <dt>
- * HTTPSearcher.LOG_PORT
- * <dd>
- * The port used for the request URI sent.
- * <dt>
- * HTTPSearcher.LOG_PATH
- * <dd>
- * Path element of the request URI sent.
- * <dt>
- * HTTPSearcher.LOG_STATUS
- * <dd>
- * Status code of the HTTP response.
- * <dt>
- * HTTPSearcher.LOG_PROXY_TYPE
- * <dd>
- * The proxy type used, if any. Default is "http".
- * <dt>
- * HTTPSearcher.LOG_PROXY_HOST
- * <dd>
- * The proxy host, if any.
- * <dt>
- * HTTPSearcher.LOG_PROXY_PORT
- * <dd>
- * The proxy port, if any.
- * <dt>
- * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name
- * <dd>
- * The content of any additional request header fields.
- * <dt>
- * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name
- * <dd>
- * The content of any additional response header fields.
- * </dl>
-
- * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a>
- * @author bratseth
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public abstract class HTTPClientSearcher extends HTTPSearcher {
-
- static final CompoundName REQUEST_META_CARRIER = new CompoundName("com.yahoo.search.federation.http.HTTPClientSearcher_requestMeta");
-
- protected final static Logger log = Logger.getLogger(HTTPClientSearcher.class.getName());
-
- /**
- * Creates a client searcher
- *
- * @param id the id of this instance
- * @param connections the connections this will load balance and fail over between
- * @param path the path portion of the url to be used
- */
- public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics) {
- super(id, connections, path, statistics);
- }
-
- public HTTPClientSearcher(ComponentId id, List<Connection> connections,String path,Statistics statistics,
- CertificateStore certificateStore) {
- super(id, connections, path, statistics, certificateStore);
- }
-
- public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters, Statistics statistics) {
- super(id, connections, parameters, statistics);
- }
- /**
- * Creates a client searcher
- *
- * @param id the id of this instance
- * @param connections the connections this will load balance and fail over between
- * @param parameters the parameters to use when making http calls
- * @param certificateStore the certificate store to use to pass certificates in requests
- */
- public HTTPClientSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters,
- Statistics statistics, CertificateStore certificateStore) {
- super(id, connections, parameters, statistics, certificateStore);
- }
-
- /** Overridden to avoid interfering with errors from nested searchers, which is inappropriate for a <i>client</i> */
- @Override
- public Result robustSearch(Query query, Execution execution, Connection connection) {
- return search(query,execution,connection);
- }
-
- /** Implements a search towards the connection chosen by the cluster searcher for this query */
- @Override
- public Result search(Query query, Execution execution, Connection connection) {
- Hit requestMeta = doHttpRequest(query, connection);
- Result result = execution.search(query);
- result.hits().add(requestMeta);
- return result;
- }
-
- private Hit doHttpRequest(Query query, Connection connection) {
- URI uri;
- // Create default meta hit for holding logging information
- Hit requestMeta = createRequestMeta();
- query.properties().set(REQUEST_META_CARRIER, requestMeta);
-
- query.trace("Created request information hit",false,9);
- try {
- uri = getURI(query, connection);
- } catch (MalformedURLException e) {
- query.errors().add(createMalformedUrlError(query,e));
- return requestMeta;
- } catch (URISyntaxException e) {
- query.errors().add(createMalformedUrlError(query,e));
- return requestMeta;
- }
-
- HttpEntity entity;
- try {
- if (query.getTraceLevel()>=1)
- query.trace("Fetching " + uri.toString(), false, 1);
- entity = getEntity(uri, requestMeta, query);
- } catch (IOException e) {
- query.errors().add(ErrorMessage.createBackendCommunicationError(
- "Error when trying to connect to HTTP backend in " + this + " using " + connection + " for " +
- query + ": " + Exceptions.toMessageString(e)));
- return requestMeta;
- } catch (TimeoutException e) {
- query.errors().add(ErrorMessage.createTimeout("HTTP traffic timed out in "
- + this + " for " + query + ": " + e.getMessage()));
- return requestMeta;
- }
- if (entity==null) {
- query.errors().add(ErrorMessage.createBackendCommunicationError(
- "No result from connecting to HTTP backend in " + this + " using " + connection + " for " + query));
- return requestMeta;
- }
-
- try {
- query = handleResponse(entity,query);
- }
- catch (IOException e) {
- query.errors().add(ErrorMessage.createBackendCommunicationError(
- "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e)));
- } finally {
- cleanupHttpEntity(entity);
- }
- return requestMeta;
- }
-
- /** Overrides to pass the query on to the next searcher */
- @Override
- public Result search(Query query, Execution execution, ErrorMessage error) {
- query.errors().add(error);
- return execution.search(query);
- }
-
- /** Do nothing on fill in client searchers */
- @Override
- public void fill(Result result,String summaryClass,Execution execution,Connection connection) {
- }
-
- /**
- * Convenience hook for unmarshalling the response and adding the information to the query.
- * Implement this or <code>handleResponse(entity,query)</code> in any subclass.
- * This default implementation throws an exception.
- *
- * @param inputStream the stream containing the data from the http service
- * @param contentLength the length of the content in the stream in bytes, or a negative number if not known
- * @param query the current query, to which information from the stream should be added
- * @return query the query to propagate down the chain. This should almost always be the
- * query instance given as a parameter.
- */
- public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException {
- throw new UnsupportedOperationException("handleResponse must be implemented by " + this);
- }
-
- /**
- * Unmarshals the response and adds the resulting data to the given query.
- * This default implementation calls
- * <code>return handleResponse(entity.getContent(), entity.getContentLength(), query);</code>
- * (and does some detailed query tracing).
- *
- * @param query the current query, to which information from the stream should be added
- * @return query the query to propagate down the chain. This should almost always be the
- * query instance given as a parameter.
- */
- public Query handleResponse(HttpEntity entity, Query query) throws IOException {
- long len = entity.getContentLength();
- if (query.getTraceLevel()>=4)
- query.trace("Received " + len + " bytes response in " + this, false, 4);
- query = handleResponse(entity.getContent(), len, query);
- if (query.getTraceLevel()>=2)
- query.trace("Handled " + len + " bytes response in " + this, false, 2);
- return query;
- }
-
- /** Never retry individual queries to clients for now */
- @Override
- protected boolean shouldRetry(Query query,Result result) { return false; }
-
- /**
- * numHits and offset should not be part of the cache key as cache supports
- * partial read/write that is only one cache entry is maintained per query
- * irrespective of the offset and numhits.
- */
- public abstract Map<String, String> getCacheKey(Query q);
-
- /**
- * Adds all key-values starting by "service." + getClientName() in query.properties().
- * Returns the empty map if {@link #getServiceName} is not overridden.
- */
- @Override
- public Map<String,String> getQueryMap(Query query) {
- LinkedHashMap<String, String> queryMap=new LinkedHashMap<>();
- if (getServiceName().isEmpty()) return queryMap;
-
- for (Map.Entry<String,Object> objectProperty : query.properties().listProperties("service." + getServiceName()).entrySet()) // TODO: Make more efficient using CompoundName
- queryMap.put(objectProperty.getKey(),objectProperty.getValue().toString());
- return queryMap;
- }
-
- /**
- * Override this to return the name of the service this is a client of.
- * This is used to look up service specific properties as service.getServiceName.serviceSpecificProperty.
- * This default implementation returns "", which means service specific parameters will not be used.
- */
- protected String getServiceName() { return ""; }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java
deleted file mode 100644
index a7b32678c1a..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPParameters.java
+++ /dev/null
@@ -1,318 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.google.common.base.Preconditions;
-import com.yahoo.search.federation.ProviderConfig.PingOption;
-import org.apache.http.conn.params.ConnManagerParams;
-import org.apache.http.conn.params.ConnPerRouteBean;
-import org.apache.http.params.BasicHttpParams;
-import org.apache.http.params.HttpConnectionParams;
-import org.apache.http.params.HttpParams;
-
-import com.yahoo.search.federation.ProviderConfig;
-
-/**
- * A set of parameters for talking to an http backend
- *
- * @author bratseth
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public final class HTTPParameters {
-
- public static final String RETRIES = "com.yahoo.search.federation.http.retries";
-
- private boolean frozen=false;
-
- // All timing parameters below are in milliseconds
- /** The url request path portion */
- private String path="/";
- private int connectionTimeout=2000;
- private int readTimeout=5000;
- private boolean persistentConnections=true;
- private boolean enableProxy = false;
- private String proxyHost = "localhost";
- private int proxyPort = 1080;
- private String method = "GET";
- private String schema = "http";
- private String inputEncoding = "utf-8";
- private String outputEncoding = "utf-8";
- private int maxTotalConnections=10000;
- private int maxConnectionsPerRoute=10000;
- private int socketBufferSizeBytes=-1;
- private int retries = 1;
- private int configuredReadTimeout = -1;
- private int configuredConnectionTimeout = -1;
- private int connectionPoolTimeout = -1;
- private String certificateProxy = null;
- private int certificatePort = 0;
- private String certificateApplicationId = null;
- private boolean certificateUseProxy = false;
- private long certificateTtl = 0L;
- private long certificateRetry = 0L;
-
- private PingOption.Enum pingOption = PingOption.NORMAL;
-
-
- private boolean followRedirects = true;
-
- public HTTPParameters() {}
-
- public HTTPParameters(String path) {
- setPath(path);
- }
-
- public HTTPParameters(ProviderConfig providerConfig) {
- configuredReadTimeout = (int) (providerConfig.readTimeout() * 1000.0d);
- configuredConnectionTimeout = (int) (providerConfig.connectionTimeout() * 1000.0d);
- connectionPoolTimeout = (int) (providerConfig.connectionPoolTimeout() * 1000.0d);
- retries = providerConfig.retries();
- setPath(providerConfig.path());
- certificateUseProxy = providerConfig.yca().useProxy();
- if (certificateUseProxy) {
- certificateProxy = providerConfig.yca().host();
- certificatePort = providerConfig.yca().port();
- }
- certificateApplicationId = providerConfig.yca().applicationId();
- certificateTtl = providerConfig.yca().ttl() * 1000L;
- certificateRetry = providerConfig.yca().retry() * 1000L;
- followRedirects = providerConfig.followRedirects();
- pingOption = providerConfig.pingOption();
- }
-
- /**
- * Set the url path to use in queries to this. If the argument is null or empty the path is set to "/".
- * If a leading "/" is missing, it is added automatically.
- */
- public final void setPath(String path) {
- if (path==null || path.isEmpty()) path="/";
-
- if (! path.startsWith("/"))
- path="/" + path;
- this.path = path;
- }
-
- public PingOption.Enum getPingOption() {
- return pingOption;
- }
-
- public void setPingOption(PingOption.Enum pingOption) {
- Preconditions.checkNotNull(pingOption);
- ensureNotFrozen();
- this.pingOption = pingOption;
- }
-
- /** Returns the url path. Default is "/". */
- public String getPath() { return path; }
-
- public boolean getFollowRedirects() {
- return followRedirects;
- }
-
- public void setFollowRedirects(boolean followRedirects) {
- ensureNotFrozen();
- this.followRedirects = followRedirects;
- }
-
-
- public void setConnectionTimeout(int connectionTimeout) {
- ensureNotFrozen();
- this.connectionTimeout=connectionTimeout;
- }
-
- /** Returns the connection timeout in milliseconds. Default is 2000. */
- public int getConnectionTimeout() { return connectionTimeout; }
-
- public void setReadTimeout(int readTimeout) {
- ensureNotFrozen();
- this.readTimeout=readTimeout;
- }
-
- /** Returns the read timeout in milliseconds. Default is 5000. */
- public int getReadTimeout() { return readTimeout; }
-
- /**
- * <b>Note: This is currently largely a noop: Connections are reused even when this is set to true.
- * The setting will change from sharing connections between threads to only reusing it within a thread
- * but it is still reused.</b>
- */
- public void setPersistentConnections(boolean persistentConnections) {
- ensureNotFrozen();
- this.persistentConnections=persistentConnections;
- }
-
- /** Returns whether this should use persistent connections. Default is true. */
- public boolean getPersistentConnections() { return persistentConnections; }
-
- /** Returns whether proxying should be enabled. Default is false. */
- public boolean getEnableProxy() { return enableProxy; }
-
- public void setEnableProxy(boolean enableProxy ) {
- ensureNotFrozen();
- this.enableProxy=enableProxy;
- }
-
- /** Returns the proxy type to use (if enabled). Default is "http". */
- public String getProxyType() {
- return "http";
- }
-
- public void setProxyHost(String proxyHost) {
- ensureNotFrozen();
- this.proxyHost=proxyHost;
- }
-
- /** Returns the proxy host to use (if enabled). Default is "localhost". */
- public String getProxyHost() { return proxyHost; }
-
- public void setProxyPort(int proxyPort) {
- ensureNotFrozen();
- this.proxyPort=proxyPort;
- }
-
- /** Returns the proxy port to use (if enabled). Default is 1080. */
- public int getProxyPort() { return proxyPort; }
-
- public void setMethod(String method) {
- ensureNotFrozen();
- this.method=method;
- }
-
- /** Returns the http method to use. Default is "GET". */
- public String getMethod() { return method; }
-
- public void setSchema(String schema) {
- ensureNotFrozen();
- this.schema=schema;
- }
-
- /** Returns the schema to use. Default is "http". */
- public String getSchema() { return schema; }
-
- public void setInputEncoding(String inputEncoding) {
- ensureNotFrozen();
- this.inputEncoding=inputEncoding;
- }
-
- /** Returns the input encoding. Default is "utf-8". */
- public String getInputEncoding() { return inputEncoding; }
-
- public void setOutputEncoding(String outputEncoding) {
- ensureNotFrozen();
- this.outputEncoding=outputEncoding;
- }
-
- /** Returns the output encoding. Default is "utf-8". */
- public String getOutputEncoding() { return outputEncoding; }
-
- /** Make this unmodifiable. Note that any thread synchronization must be done outside this object. */
- public void freeze() {
- frozen=true;
- }
-
- private void ensureNotFrozen() {
- if (frozen) throw new IllegalStateException("Cannot modify frozen " + this);
- }
-
- /**
- * Returns the eligible subset of this as a HttpParams snapshot
- * AND configures the Apache HTTP library with the parameters of this
- */
- public HttpParams toHttpParams() {
- return toHttpParams(connectionTimeout, readTimeout);
- }
-
- /**
- * Returns the eligible subset of this as a HttpParams snapshot
- * AND configures the Apache HTTP library with the parameters of this
- */
- public HttpParams toHttpParams(int connectionTimeout, int readTimeout) {
- HttpParams params = new BasicHttpParams();
- // force use of configured value if available
- if (configuredConnectionTimeout > 0) {
- HttpConnectionParams.setConnectionTimeout(params, configuredConnectionTimeout);
- } else {
- HttpConnectionParams.setConnectionTimeout(params, connectionTimeout);
- }
- if (configuredReadTimeout > 0) {
- HttpConnectionParams.setSoTimeout(params, configuredReadTimeout);
- } else {
- HttpConnectionParams.setSoTimeout(params, readTimeout);
- }
- if (socketBufferSizeBytes > 0) {
- HttpConnectionParams.setSocketBufferSize(params, socketBufferSizeBytes);
- }
- if (connectionPoolTimeout > 0) {
- ConnManagerParams.setTimeout(params, connectionPoolTimeout);
- }
- ConnManagerParams.setMaxTotalConnections(params, maxTotalConnections);
- ConnManagerParams.setMaxConnectionsPerRoute(params, new ConnPerRouteBean(maxConnectionsPerRoute));
- if (retries >= 0) {
- params.setIntParameter(RETRIES, retries);
- }
- params.setParameter("http.protocol.handle-redirects", followRedirects);
- return params;
- }
-
- public int getMaxTotalConnections() {
- return maxTotalConnections;
- }
-
- public void setMaxTotalConnections(int maxTotalConnections) {
- ensureNotFrozen();
- this.maxTotalConnections = maxTotalConnections;
- }
-
- public int getMaxConnectionsPerRoute() {
- return maxConnectionsPerRoute;
- }
-
- public void setMaxConnectionsPerRoute(int maxConnectionsPerRoute) {
- ensureNotFrozen();
- this.maxConnectionsPerRoute = maxConnectionsPerRoute;
- }
-
- public int getSocketBufferSizeBytes() {
- return socketBufferSizeBytes;
- }
-
- public void setSocketBufferSizeBytes(int socketBufferSizeBytes) {
- ensureNotFrozen();
- this.socketBufferSizeBytes = socketBufferSizeBytes;
- }
-
- public int getRetries() {
- return retries;
- }
-
- public void setRetries(int retries) {
- ensureNotFrozen();
- this.retries = retries;
- }
-
- public String getYcaProxy() {
- return certificateProxy;
- }
-
- public int getYcaPort() {
- return certificatePort;
- }
-
- public String getYcaApplicationId() {
- return certificateApplicationId;
- }
-
- public boolean getYcaUseProxy() {
- return certificateUseProxy;
- }
-
- public long getYcaTtl() {
- return certificateTtl;
- }
-
- public long getYcaRetry() {
- return certificateRetry;
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java
deleted file mode 100644
index c72c2f26a1c..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPProviderSearcher.java
+++ /dev/null
@@ -1,261 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.google.common.collect.ImmutableList;
-import com.yahoo.component.ComponentId;
-import com.yahoo.jdisc.http.CertificateStore;
-import com.yahoo.search.cache.QrBinaryCacheConfig;
-import com.yahoo.search.cache.QrBinaryCacheRegionConfig;
-import com.yahoo.yolean.Exceptions;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.federation.FederationSearcher;
-import com.yahoo.search.query.Properties;
-import com.yahoo.search.result.ErrorMessage;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.statistics.Counter;
-import com.yahoo.statistics.Statistics;
-import com.yahoo.statistics.Value;
-
-import org.apache.http.HttpEntity;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.*;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Superclass of searchers which talks to HTTP backends. Implement a subclass to talk to a backend
- * over HTTP which is not supported by the platform out of the box.
- * <p>
- * Implementations must override one of the <code>unmarshal</code> methods to unmarshal the response.
- * </p>
- *
- * @author Arne Bergene Fossaa
- * @author bratseth
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public abstract class HTTPProviderSearcher extends HTTPSearcher {
-
- private final Counter emptyResults;
- private final Value hitsPerQuery;
- private final Value responseLatency;
- private final Counter readTimeouts;
-
- private final static List<String> excludedSourceProperties = ImmutableList.of("offset", "hits", "provider");
-
- protected final static Logger log = Logger.getLogger(HTTPProviderSearcher.class.getName());
-
- /** The name of the cache used (which is just getid().stringValue(), or null if no cache is used */
- protected String cacheName=null;
-
- public HTTPProviderSearcher(ComponentId id, List<Connection> connections,String path, Statistics statistics) {
- this(id,connections,new HTTPParameters(path), statistics);
- }
-
- /** Creates a http provider searcher using id.getName as provider name */
- public HTTPProviderSearcher(ComponentId id, List<Connection> connections, String path,
- Statistics statistics, CertificateStore certificateStore) {
- this(id, connections, new HTTPParameters(path), statistics, certificateStore);
- }
-
- public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters,
- Statistics statistics) {
- this(id, connections, parameters, statistics, new ThrowingCertificateStore());
- }
-
- /**
- * Creates a provider searcher
- *
- * @param id the id of this instance
- * @param connections the connections this will load balance and fail over between
- * @param parameters the parameters to use when making http calls
- */
- public HTTPProviderSearcher(ComponentId id, List<Connection> connections, HTTPParameters parameters,
- Statistics statistics, CertificateStore certificateStore) {
- super(id, connections, parameters, statistics, certificateStore);
- String suffix = "_" + getId().getName().replace('.', '_');
- hitsPerQuery = new Value("hits_per_query" + suffix, statistics,
- new Value.Parameters().setLogRaw(false).setNameExtension(false).setLogMean(true));
- responseLatency = new Value(LOG_LATENCY_START + suffix, statistics,
- new Value.Parameters().setLogRaw(false).setLogMean(true).setNameExtension(false));
- emptyResults = new Counter("empty_results" + suffix, statistics, false);
- readTimeouts = new Counter(LOG_READ_TIMEOUT_PREFIX + suffix, statistics, false);
- }
-
- /** @deprecated this method does nothing */
- @Deprecated // OK
- protected void configureCache(final QrBinaryCacheConfig cacheConfig,final QrBinaryCacheRegionConfig regionConfig) {
- }
-
- /**
- * Unmarshal the stream by converting it to hits and adding the hits to the given result.
- * A convenience hook called by the default <code>unmarshal(entity,result).</code>
- * Override this in subclasses which does not override <code>unmarshal(entity,result).</code>
- * <p>
- * This default implementation throws an exception.
- *
- * @param stream the stream of data returned
- * @param contentLength the length of the content in bytes if known, or a negative number if unknown
- * @param result the result to which unmarshalled data should be added
- */
- public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException {
- throw new UnsupportedOperationException("Unmarshal must be implemented by " + this);
- }
-
- /**
- * Unmarshal the result from an http entity. This default implementation calls
- * <code>unmarshal(entity.getContent(), entity.getContentLength(), result)</code>
- * (and does some detailed query tracing).
- *
- * @param entity the entity containing the data to unmarshal
- * @param result the result to which unmarshalled data should be added
- */
- public void unmarshal(HttpEntity entity,Result result) throws IOException {
- Query query=result.getQuery();
- long len = entity.getContentLength();
- if (query.getTraceLevel()>=4)
- query.trace("Received " + len + " bytes response in " + this, false, 4);
- query.trace("Unmarshaling result.", false, 6);
- unmarshal(entity.getContent(), len, result);
-
- if (query.getTraceLevel()>=2)
- query.trace("Handled " + len + " bytes response in " + this, false, 2);
-
- }
-
- protected void addNonExcludedSourceProperties(Query query, Map<String, String> queryMap) {
- Properties sourceProperties = FederationSearcher.getSourceProperties(query);
- if (sourceProperties != null) {
- for(Map.Entry<String, Object> entry : sourceProperties.listProperties("").entrySet()) {
- if (!excludedSourceProperties.contains(entry.getKey())) {
- queryMap.put(entry.getKey(), entry.getValue().toString());
- }
- }
- }
- }
-
- /**
- * Hook called at the moment the result is returned from this searcher. This default implementation
- * does <code>return result</code>.
- *
- * @param result the result which is to be returned
- * @param requestMeta the request information hit, or null if none was created (e.g if this was a cache lookup)
- * @param e the exception caused during execution of this query, or null if none
- * @return the result which is returned upwards
- */
- protected Result inspectAndReturnFinalResult(Result result, Hit requestMeta, Exception e) {
- return result;
- }
-
- private Result statisticsBeforeInspection(Result result, Hit requestMeta, Exception e) {
- int hitCount = result.getConcreteHitCount();
- if (hitCount == 0) {
- emptyResults.increment();
- }
- hitsPerQuery.put((double) hitCount);
-
- if (requestMeta != null) {
- requestMeta.setField(LOG_HITCOUNT, Integer.valueOf(hitCount));
- }
-
- return inspectAndReturnFinalResult(result, requestMeta, e);
- }
-
-
- @Override
- protected void logResponseLatency(long latency) {
- responseLatency.put((double) latency);
- }
-
- @Override
- public Result search(Query query, Execution execution,Connection connection) {
- // Create default meta hit for holding logging information
- Hit requestMeta = createRequestMeta();
- Result result = new Result(query);
- result.hits().add(requestMeta);
- query.trace("Created request information hit", false, 9);
-
- try {
- URI uri = getURI(query, requestMeta, connection);
- if (query.getTraceLevel()>=1)
- query.trace("Fetching " + uri.toString(), false, 1);
- long requestStartTime = System.currentTimeMillis();
-
- HttpEntity entity = getEntity(uri, requestMeta, query);
-
- // Why should consumeEntity call inspectAndReturnFinalResult itself?
- // Seems confusing to me.
- return entity == null
- ? statisticsBeforeInspection(result, requestMeta, null)
- : consumeEntity(entity, query, result, requestMeta, requestStartTime);
-
- } catch (MalformedURLException|URISyntaxException e) {
- result.hits().addError(createMalformedUrlError(query,e));
- return statisticsBeforeInspection(result, requestMeta, e);
- } catch (TimeoutException e) {
- result.hits().addError(ErrorMessage.createTimeout("No time left for HTTP traffic in "
- + this
- + " for " + query + ": " + e.getMessage()));
- return statisticsBeforeInspection(result, requestMeta, e);
- } catch (IOException e) {
- result.hits().addError(ErrorMessage.createBackendCommunicationError(
- "Error when trying to connect to HTTP backend in " + this
- + " for " + query + ": " + Exceptions.toMessageString(e)));
- return statisticsBeforeInspection(result, requestMeta, e);
- }
- }
-
- private Result consumeEntity(HttpEntity entity, Query query, Result result, Hit logHit, long requestStartTime) {
-
- try {
- // remove some time from timeout to allow for close calls with return result
- unmarshal(new TimedHttpEntity(entity, query.getStartTime(), Math.max(1, query.getTimeout() - 10)), result);
- logHit.setField(LOG_LATENCY_FINISH, System.currentTimeMillis() - requestStartTime);
- return statisticsBeforeInspection(result, logHit, null);
- } catch (IOException e) {
- result.hits().addError(ErrorMessage.createBackendCommunicationError(
- "Error when trying to consume input in " + this + ": " + Exceptions.toMessageString(e)));
- return statisticsBeforeInspection(result, logHit, e);
- } catch (TimeoutException e) {
- readTimeouts.increment();
- result.hits().addError(ErrorMessage
- .createTimeout("Timed out while reading/unmarshaling from backend in "
- + this + " for " + query
- + ": " + e.getMessage()));
- return statisticsBeforeInspection(result, logHit, e);
- } finally { // TODO: The scope of this finally must be enlarged to release the connection also on errors
- cleanupHttpEntity(entity);
- }
- }
-
- /**
- * Returns the key-value pairs that should be added as properties to the request url sent to the service.
- * Must be overridden in subclasses to add the key-values expected by the service in question, unless
- * {@link #getURI} (from which this is called) is overridden.
- * <p>
- * This default implementation returns the query.properties() prefixed by
- * "source.[sourceName]" or "property.[propertyName]"
- * (by calling {@link #addNonExcludedSourceProperties}).
- */
- @Override
- public Map<String,String> getQueryMap(Query query) {
- Map<String,String> queryMap = super.getQueryMap(query);
- addNonExcludedSourceProperties(query, queryMap);
- return queryMap;
- }
-
- /**
- * @deprecated the cache key is ignored as there is no built-in caching support
- */
- @Deprecated // OK
- public abstract Map<String, String> getCacheKey(Query q);
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java
deleted file mode 100644
index edf347bd84e..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/HTTPSearcher.java
+++ /dev/null
@@ -1,961 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.google.inject.Inject;
-import com.yahoo.component.ComponentId;
-import com.yahoo.jdisc.http.CertificateStore;
-import com.yahoo.log.LogLevel;
-import com.yahoo.prelude.Ping;
-import com.yahoo.prelude.Pong;
-import com.yahoo.yolean.Exceptions;
-import com.yahoo.search.Query;
-import com.yahoo.search.cluster.ClusterSearcher;
-import com.yahoo.search.federation.ProviderConfig.PingOption;
-import com.yahoo.search.result.ErrorMessage;
-import com.yahoo.search.result.Hit;
-import com.yahoo.statistics.Counter;
-import com.yahoo.statistics.Statistics;
-import com.yahoo.text.Utf8;
-
-import org.apache.http.*;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.HttpRequestRetryHandler;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.conn.ClientConnectionManager;
-import org.apache.http.conn.ConnectTimeoutException;
-import org.apache.http.conn.params.ConnManagerParams;
-import org.apache.http.conn.params.ConnRoutePNames;
-import org.apache.http.conn.routing.HttpRoutePlanner;
-import org.apache.http.conn.scheme.PlainSocketFactory;
-import org.apache.http.conn.scheme.Scheme;
-import org.apache.http.conn.scheme.SchemeRegistry;
-import org.apache.http.conn.ssl.SSLSocketFactory;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.http.impl.conn.DefaultHttpRoutePlanner;
-import org.apache.http.impl.conn.SingleClientConnManager;
-import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
-import org.apache.http.params.HttpParams;
-import org.apache.http.params.HttpProtocolParams;
-import org.apache.http.protocol.BasicHttpContext;
-import org.apache.http.protocol.ExecutionContext;
-import org.apache.http.protocol.HttpContext;
-import org.apache.http.protocol.HttpRequestExecutor;
-import org.apache.http.util.EntityUtils;
-
-import javax.net.ssl.SSLHandshakeException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.io.UnsupportedEncodingException;
-import java.net.*;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.TimeUnit;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Generic superclass of searchers making connections to some HTTP service. This
- * supports clustered connections - a list of alternative servers may be given,
- * requests will be hashed across these and failed over in case some are down.
- * <p>
- * This simply provides some utility methods for working with http connections
- * and implements ping against the service.
- *
- * <p>This searcher contains code from the Apache httpcomponents client library,
- * licensed to the Apache Software Foundation under the Apache License, Version
- * 2.0. Please refer to http://www.apache.org/licenses/LICENSE-2.0 for details.
- *
- * <p>This class automatically adds a meta hit containing latency and other
- * meta information about the obtained HTTP data using createRequestMeta().
- * The fields available in the hit are:</p>
- *
- * <dl><dt>
- * HTTPSearcher.LOG_LATENCY_START
- * <dd>
- * The latency of the external provider answering a request.
- * <dt>
- * HTTPSearcher.LOG_LATENCY_FINISH
- * <dd>
- * Total time of the HTTP traffic, but also decoding of the data, as this
- * happens at the same time.
- * <dt>
- * HTTPSearcher.LOG_HITCOUNT
- * <dd>
- * Number of concrete hits in the result returned by this provider.
- * <dt>
- * HTTPSearcher.LOG_URI
- * <dd>
- * The complete URI used for external service.
- * <dt>
- * HTTPSearcher.LOG_SCHEME
- * <dd>
- * The scheme of the request URI sent.
- * <dt>
- * HTTPSearcher.LOG_HOST
- * <dd>
- * The host used for the request URI sent.
- * <dt>
- * HTTPSearcher.LOG_PORT
- * <dd>
- * The port used for the request URI sent.
- * <dt>
- * HTTPSearcher.LOG_PATH
- * <dd>
- * Path element of the request URI sent.
- * <dt>
- * HTTPSearcher.LOG_STATUS
- * <dd>
- * Status code of the HTTP response.
- * <dt>
- * HTTPSearcher.LOG_PROXY_TYPE
- * <dd>
- * The proxy type used, if any. Default is "http".
- * <dt>
- * HTTPSearcher.LOG_PROXY_HOST
- * <dd>
- * The proxy host, if any.
- * <dt>
- * HTTPSearcher.LOG_PROXY_PORT
- * <dd>
- * The proxy port, if any.
- * <dt>
- * HTTPSearcher.LOG_HEADER_PREFIX prepended to request header field name
- * <dd>
- * The content of any additional request header fields.
- * <dt>
- * HTTPSearcher.LOG_RESPONSE_HEADER_PREFIX prepended to response header field name
- * <dd>
- * The content of any additional response header fields.
- * </dl>
- *
- * @author Arne Bergene Fossaa
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public abstract class HTTPSearcher extends ClusterSearcher<Connection> {
-
- protected static final String YCA_HTTP_HEADER = "Yahoo-App-Auth";
-
- private static final Charset iso8859Charset = Charset.forName("ISO-8859-1");
-
- // Logging field name constants
- public static final String LOG_PATH = "path";
- public static final String LOG_PORT = "port";
- public static final String LOG_HOST = "host";
- public static final String LOG_IP_ADDRESS = "ip_address";
- public static final String IP_ADDRESS_UNKNOWN = "unknown";
-
- public static final String LOG_SCHEME = "scheme";
- public static final String LOG_URI = "uri";
- public static final String LOG_PROXY_PORT = "proxy_port";
- public static final String LOG_PROXY_HOST = "proxy_host";
- public static final String LOG_PROXY_TYPE = "proxy_type";
- public static final String LOG_STATUS = "status";
- public static final String LOG_LATENCY_FINISH = "latency_finish";
- public static final String LOG_LATENCY_START = "latency_start";
- public static final String LOG_LATENCY_CONNECT = "latency_connect";
- public static final String LOG_QUERY_PARAM_PREFIX = "query_param_";
- public static final String LOG_HEADER_PREFIX = "header_";
- public static final String LOG_RESPONSE_HEADER_PREFIX = "response_header_";
- public static final String LOG_HITCOUNT = "hit_count";
- public static final String LOG_CONNECT_TIMEOUT_PREFIX = "connect_timeout_";
- public static final String LOG_READ_TIMEOUT_PREFIX = "read_timeout_";
-
- protected final Logger log = Logger.getLogger(HTTPSearcher.class.getName());
-
- /** The HTTP parameters to use. Assigned in the constructor */
- private HTTPParameters httpParameters;
-
- private final Counter connectTimeouts;
-
- /** Whether to use certificates */
- protected boolean useCertificate = false;
-
- private final CertificateStore certificateStore;
-
- /** The (optional) certificate application ID. */
- private String certificateApplicationId = null;
-
- /** The (optional) certificate server proxy */
- protected HttpHost certificateProxy = null;
-
- /** Certificate cache TTL in ms */
- private long certificateTtl = 0L;
-
- /** Certificate server retry rate in the cache if no cert is found, in ms */
- private long certificateRetry = 0L;
-
- /** Set at construction if this is using persistent connections */
- private ClientConnectionManager sharedConnectionManager = null;
-
- /** Set at construction if using non-persistent connections */
- private ThreadLocal<SingleClientConnManager> singleClientConnManagerThreadLocal = null;
-
- private static final SchemeRegistry schemeRegistry = new SchemeRegistry();
-
- static {
- schemeRegistry.register(new Scheme("http", PlainSocketFactory
- .getSocketFactory(), 80));
- schemeRegistry.register(new Scheme("https", SSLSocketFactory
- .getSocketFactory(), 443));
- }
-
- public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics) {
- this(componentId, connections, new HTTPParameters(path), statistics, new ThrowingCertificateStore());
- }
-
- /** Creates a http searcher with default connection and read timeouts (currently 2 and 5s respectively) */
- public HTTPSearcher(ComponentId componentId, List<Connection> connections,String path, Statistics statistics,
- CertificateStore certificateStore) {
- this(componentId, connections, new HTTPParameters(path), statistics, certificateStore);
- }
-
- public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters,
- Statistics statistics) {
- this(componentId, connections, parameters, statistics, new ThrowingCertificateStore());
- }
- /**
- * Creates a http searcher
- *
- * @param componentId the id of this instance
- * @param connections the connections to establish to the backend nodes
- * @param parameters the http parameters to use. This object will be frozen if it isn't already
- */
- @Inject
- public HTTPSearcher(ComponentId componentId, List<Connection> connections, HTTPParameters parameters,
- Statistics statistics, CertificateStore certificateStore) {
- super(componentId,connections,false);
- String suffix = "_" + getId().getName().replace('.', '_');
-
- connectTimeouts = new Counter(LOG_CONNECT_TIMEOUT_PREFIX + suffix, statistics, false);
-
- parameters.freeze();
- this.httpParameters = parameters;
- this.certificateStore = certificateStore;
-
- if (parameters.getPersistentConnections()) {
- HttpParams params=parameters.toHttpParams();
- HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
- ConnManagerParams.setTimeout(params, 10);
- sharedConnectionManager = new ThreadSafeClientConnManager(params, schemeRegistry);
- Thread connectionPurgerThread = new Thread(() -> {
- //this is the default value in yahoo jvm installations
- long DNSTTLSec = 120;
- while (true) {
- try {
- Thread.sleep(DNSTTLSec * 1000);
- if (sharedConnectionManager == null)
- continue;
-
- sharedConnectionManager.closeExpiredConnections();
- DNSTTLSec = Long.valueOf(java.security.Security
- .getProperty("networkaddress.cache.ttl"));
- //No DNS TTL, no need to close idle connections
- if (DNSTTLSec <= 0) {
- DNSTTLSec = 120;
- continue;
- }
- sharedConnectionManager.closeIdleConnections(2 * DNSTTLSec, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- return;
- } catch (NumberFormatException e) {
- continue;
- }
- }
- });
- connectionPurgerThread.setDaemon(true);
- connectionPurgerThread.start();
-
- }
- else {
- singleClientConnManagerThreadLocal =new ThreadLocal<>();
- }
-
- initializeCertificate(httpParameters, certificateStore);
- }
-
- /**
- * Initialize certificate store and proxy if they have been set to non-null,
- * non-empty values. It will wrap thrown exceptions from the certificate store into
- * RuntimeException and propagate them.
- */
- private void initializeCertificate(HTTPParameters parameters, CertificateStore certificateStore) {
- String applicationId = parameters.getYcaApplicationId();
- String proxy = parameters.getYcaProxy();
- int port = parameters.getYcaPort();
- long ttl = parameters.getYcaTtl();
- long retry = parameters.getYcaRetry();
-
- if (applicationId != null && !applicationId.trim().isEmpty()) {
- initializeCertificate(applicationId, ttl, retry, certificateStore);
- }
-
- if (parameters.getYcaUseProxy()) {
- initializeProxy(proxy, port);
- }
- }
-
- /** Returns the HTTP parameters used in this. This is always frozen */
- public HTTPParameters getParameters() { return httpParameters; }
-
- /**
- * Returns the key-value pairs that should be added as properties to the request url sent to the service.
- * Must be overridden in subclasses to add the key-values expected by the service in question, unless
- * {@link #getURI} (from which this is called) is overridden.
- * <p>
- * This default implementation returns an empty LinkedHashMap.
- */
- public Map<String,String> getQueryMap(Query query) {
- return new LinkedHashMap<>();
- }
-
- /**
- * Initialize the certificate.
- * This will warn but not throw if certificates could not be loaded, as the certificates
- * are external state which can fail independently.
- */
- private void initializeCertificate(String applicationId, long ttl, long retry, CertificateStore certificateStore) {
- try {
- // get the certificate, i.e. init the cache and check integrity
- String certificate = certificateStore.getCertificate(applicationId, ttl, retry);
- if (certificate == null) {
- getLogger().log(LogLevel.WARNING, "No certificate found for application '" + applicationId + "'");
- return;
- }
-
- this.useCertificate = true;
- this.certificateApplicationId = applicationId;
- this.certificateTtl = ttl;
- this.certificateRetry = retry;
- getLogger().log(LogLevel.CONFIG, "Got certificate: " + certificate);
- }
- catch (Exception e) {
- getLogger().log(LogLevel.WARNING,"Exception while initializing certificate for application '" +
- applicationId + "' in " + this, e);
- }
- }
-
- /**
- * Initialize the certificate proxy setting.
- */
- private void initializeProxy(String host, int port) {
- certificateProxy = new HttpHost(host, port);
- getLogger().log(LogLevel.CONFIG, "Proxy is configured; will use proxy: " + certificateProxy);
- }
-
- /**
- * Same a {@code getURI(query, offset, hits, null)}.
- * @see #getURI(Query, Hit, Connection)
- */
- protected URI getURI(Query query,Connection connection) throws MalformedURLException, URISyntaxException {
- Hit requestMeta;
- try {
- requestMeta = (Hit) query.properties().get(HTTPClientSearcher.REQUEST_META_CARRIER);
- } catch (ClassCastException e) {
- requestMeta = null;
- }
- return getURI(query, requestMeta, connection);
- }
-
- /**
- * Creates the URI for a query.
- * Populates the {@code requestMeta} meta hit with the created URI HTTP properties.
- *
- * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
- */
- protected URI getURI(Query query, Hit requestMeta, Connection connection)
- throws MalformedURLException, URISyntaxException {
- StringBuilder parameters = new StringBuilder();
-
- Map<String, String> queries = getQueryMap(query);
- if (queries.size() > 0) {
- Iterator<Map.Entry<String, String>> mapIterator = queries.entrySet().iterator();
- parameters.append("?");
- try {
- Map.Entry<String, String> entry;
- while (mapIterator.hasNext()) {
- entry = mapIterator.next();
-
- if (requestMeta != null)
- requestMeta.setField(LOG_QUERY_PARAM_PREFIX
- + entry.getKey(), entry.getValue());
-
- parameters.append(entry.getKey() + "=" + URLEncoder.encode(entry.getValue(),
- httpParameters.getInputEncoding()));
- if (mapIterator.hasNext()) {
- parameters.append("&");
- }
- }
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException("Unknown input encoding set in " + this, e);
- }
- }
-
- URI uri = new URL(httpParameters.getSchema(), connection.getHost(),
- connection.getPort(), getPath() + parameters.toString()).toURI();
- if (requestMeta != null) {
- requestMeta.setField(LOG_URI, uri.toString());
- requestMeta.setField(LOG_SCHEME, uri.getScheme());
- requestMeta.setField(LOG_HOST, uri.getHost());
- requestMeta.setField(LOG_PORT, uri.getPort());
- requestMeta.setField(LOG_PATH, uri.getPath());
- }
- return uri;
- }
-
- /**
- * Called by getURI() to get the path of the URI for the external service.
- * The default implementation returns httpParameters.getPath(); subclasses
- * which only wants to override the path from httpParameters may use this
- * method instead of overriding all of getURI().
- *
- * @return the path to use for getURI
- */
- protected String getPath() {
- return httpParameters.getPath();
- }
-
- /**
- * The URI that is used to check if the provider is up or down. This will again be used in the
- * checkPing method by checking that we get a response that has a good status code (below 300). If better
- * validation than just status code checking is needed, override the checkPing method.
- */
- protected URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException {
- return new URL(httpParameters.getSchema(),connection.getHost(),connection.getPort(),getPingPath()).toURI();
- }
-
- /**
- * Called by getPingURI() to get the path of the URI for pinging the
- * external service. The default implementation returns
- * httpParameters.getPath(); subclasses which only wants to override the
- * path from httpParameters may use this method instead of overriding all of
- * getPingURI().
- *
- * @return the path to use for getPingURI
- */
- protected String getPingPath() {
- return httpParameters.getPath();
- }
-
- /**
- * Checks if the response is valid.
- * @param response The response from the ping request
- * @param pong The pong result to return back to the calling method. This method
- * will add an error to the pong result (using addError) if the status of the HTTP response is 300 or above.
- */
- protected void checkPing(HttpResponse response, Pong pong) {
- if (response.getStatusLine().getStatusCode() >= 300) {
- pong.addError(com.yahoo.search.result.ErrorMessage.createBackendCommunicationError(
- "Got error " + response.getStatusLine().getStatusCode()
- + " when contacting backend")
- );
- }
- }
-
- /**
- * Pinging in HTTPBackend is done by creating a PING uri from http://host:port/path.
- * If this returns a status that is below 300, the ping is considered good.
- *
- * If another uri is needed for pinging, reimplement getPingURI.
- *
- * Override either this method to change how ping
- */
- @Override
- public Pong ping(Ping ping, Connection connection) {
- URI uri = null;
- Pong pong = new Pong();
- HttpResponse response = null;
-
- if (httpParameters.getPingOption() == PingOption.DISABLE)
- return pong;
-
- try {
- uri = getPingURI(connection);
- if (uri == null)
- pong.addError(ErrorMessage.createIllegalQuery("Ping uri is null"));
- if (uri.getHost()==null) {
- pong.addError(ErrorMessage.createIllegalQuery("Ping uri has no host"));
- uri=null;
- }
- } catch (MalformedURLException | URISyntaxException e) {
- pong.addError(ErrorMessage.createIllegalQuery("Malformed ping uri '" + uri + "': " +
- Exceptions.toMessageString(e)));
- } catch (RuntimeException e) {
- log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection +
- " using uri '" + uri + "'",e);
- pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " +
- Exceptions.toMessageString(e)));
- }
-
- if (uri == null) return pong;
- pong.setPingInfo("using uri '" + uri + "'");
-
- try {
- response = getPingResponse(uri, ping);
- checkPing(response, pong);
- } catch (IOException e) {
- // We do not have a valid ping
- pong.addError(ErrorMessage.createBackendCommunicationError(
- "Exception thrown when pinging with url '" + uri + "': " + Exceptions.toMessageString(e)));
- } catch (TimeoutException e) {
- pong.addError(ErrorMessage.createTimeout("Timeout for ping " + uri + " in " + this + ": " + e.getMessage()));
- } catch (RuntimeException e) {
- log.log(Level.WARNING,"Unexpected exception while attempting to ping " + connection + " using uri '" + uri + "'",e);
- pong.addError(ErrorMessage.createIllegalQuery("Unexpected problem with ping uri '" + uri + "': " +
- Exceptions.toMessageString(e)));
- } finally {
- if (response != null) {
- cleanupHttpEntity(response.getEntity());
- }
- }
-
- return pong;
- }
-
- private HttpResponse getPingResponse(URI uri, Ping ping) throws IOException {
- long timeLeft = ping.getTimeout();
- int connectionTimeout = (int) (timeLeft / 4L);
- int readTimeout = (int) (timeLeft * 3L / 4L);
-
- Map<String, String> requestHeaders = null;
- if (httpParameters.getPingOption() == PingOption.YCA)
- requestHeaders = generateYCAHeaders();
-
- return getResponse(uri, null, requestHeaders, null, connectionTimeout, readTimeout);
- }
-
- /**
- * Same a {@code getEntity(uri, null)}.
- * @param uri resource to fetch
- * @param query the originating query
- * @throws TimeoutException If query.timeLeft() equal to or lower than 0
- */
- protected HttpEntity getEntity(URI uri, Query query) throws IOException{
- return getEntity(uri, null, query);
- }
-
-
- /**
- * Gets the HTTP entity that holds the response contents.
- * @param uri the request URI.
- * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
- * @param query the originating query
- * @return the http entity, or null if none
- * @throws java.io.IOException Whenever HTTP status code is in the 300 or higher range.
- * @throws TimeoutException If query.timeLeft() equal to or lower than 0
- */
- protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException {
- if (query.getTimeLeft() <= 0) {
- throw new TimeoutException("No time left for querying external backend.");
- }
- HttpResponse response = getResponse(uri, requestMeta, query);
- StatusLine statusLine = response.getStatusLine();
-
- // Logging
- if (requestMeta != null) {
- requestMeta.setField(LOG_STATUS, statusLine.getStatusCode());
- for (HeaderIterator headers = response.headerIterator(); headers.hasNext(); ) {
- Header h = headers.nextHeader();
- requestMeta.setField(LOG_RESPONSE_HEADER_PREFIX + h.getName(), h.getValue());
- }
- }
-
- if (statusLine.getStatusCode() >= 300) {
- HttpEntity entity = response.getEntity();
- String message = createServerReporterErrorMessage(statusLine, entity);
- cleanupHttpEntity(response.getEntity());
- throw new IOException(message);
- }
-
- return response.getEntity();
- }
-
- private String createServerReporterErrorMessage(StatusLine statusLine, HttpEntity entity) {
- String message = "Error when trying to connect to HTTP backend: "
- + statusLine.getStatusCode() + " : " + statusLine.getReasonPhrase();
-
- try {
- if (entity != null) {
- message += "(Message = " + EntityUtils.toString(entity) + ")";
- }
- } catch (Exception e) {
- log.log(LogLevel.WARNING, "Could not get message.", e);
- }
-
- return message;
- }
-
- /**
- * Creates a meta hit dedicated to holding logging information. This hit has
- * the 'logging:[searcher's ID]' type.
- */
- protected Hit createRequestMeta() {
- Hit requestMeta = new Hit("logging:" + getId().toString());
- requestMeta.setMeta(true);
- requestMeta.types().add("logging");
- return requestMeta;
- }
-
- protected void cleanupHttpEntity(HttpEntity entity) {
- if (entity == null) return;
-
- try {
- entity.consumeContent();
- } catch (IOException e) {
- // It is ok if do not consume it, the resource will be freed after
- // timeout.
- // But log it just in case.
- log.log(LogLevel.getVespaLogLevel(LogLevel.DEBUG),
- "Not able to consume after processing: " + Exceptions.toMessageString(e));
- }
- }
-
- /**
- * Same as {@code getResponse(uri, null)}.
- */
- protected HttpResponse getResponse(URI uri, Query query) throws IOException{
- return getResponse(uri, null, query);
- }
-
- /**
- * Executes an HTTP request and gets the response.
- * @param uri the request URI.
- * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
- * @param query the originating query, used to calculate timeouts
- */
- protected HttpResponse getResponse(URI uri, Hit requestMeta, Query query) throws IOException {
- long timeLeft = query.getTimeLeft();
- int connectionTimeout = (int) (timeLeft / 4L);
- int readTimeout = (int) (timeLeft * 3L / 4L);
- connectionTimeout = connectionTimeout <= 0 ? 1 : connectionTimeout;
- readTimeout = readTimeout <= 0 ? 1 : readTimeout;
- HttpEntity reqEntity = getRequestEntity(query, requestMeta);
- Map<String, String> reqHeaders = getRequestHeaders(query, requestMeta);
- if ((reqEntity == null) && (reqHeaders == null)) {
- return getResponse(uri, requestMeta, connectionTimeout, readTimeout);
- } else {
- return getResponse(uri, reqEntity, reqHeaders, requestMeta, connectionTimeout, readTimeout);
- }
- }
-
- /**
- * Returns the set of headers to be passed in the http request to provider backend. The default
- * implementation returns null, unless certificates are in use. If certificates are used, it will return a map
- * only containing the needed certificate headers.
- */
- protected Map<String, String> getRequestHeaders(Query query, Hit requestMeta) {
- if (useCertificate) {
- return generateYCAHeaders();
- }
- return null;
- }
-
- /**
- * Returns the HTTP request entity to use when making the request for this query.
- * This default implementation returns null.
- *
- * <p> Do return a repeatable entity if HTTP retry is active.
- *
- * @return the http request entity to use, or null to use the default entity
- */
- protected HttpEntity getRequestEntity(Query query, Hit requestMeta) {
- return null;
- }
-
- /**
- * Executes an HTTP request and gets the response.
- * @param uri the request URI.
- * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
- * @param connectionTimeout how long to wait for getting a connection
- * @param readTimeout timeout for reading HTTP data
- */
- protected HttpResponse getResponse(URI uri, Hit requestMeta, int connectionTimeout, int readTimeout)
- throws IOException {
- return getResponse(uri, null, null, requestMeta, connectionTimeout, readTimeout);
- }
-
-
- /**
- * Executes an HTTP request and gets the response.
- * @param uri the request URI.
- * @param requestMeta a meta hit that holds logging information about this request (may be {@code null}).
- * @param connectionTimeout how long to wait for getting a connection
- * @param readTimeout timeout for reading HTTP data
- */
- protected HttpResponse getResponse(URI uri, HttpEntity reqEntity,
- Map<String, String> reqHeaders, Hit requestMeta,
- int connectionTimeout, int readTimeout) throws IOException {
-
- HttpParams httpParams = httpParameters.toHttpParams(connectionTimeout, readTimeout);
- HttpClient httpClient = createClient(httpParams);
- long start = 0L;
- HttpUriRequest request;
- if (httpParameters.getEnableProxy() && "http".equals(httpParameters.getProxyType())) {
- HttpHost proxy = new HttpHost(httpParameters.getProxyHost(),
- httpParameters.getProxyPort(), httpParameters.getProxyType());
- httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
- // Logging
- if (requestMeta != null) {
- requestMeta.setField(LOG_PROXY_TYPE, httpParameters.getProxyType());
- requestMeta.setField(LOG_PROXY_HOST, httpParameters.getProxyHost());
- requestMeta.setField(LOG_PROXY_PORT, httpParameters.getProxyPort());
- }
- }
- if (reqEntity == null) {
- request = createRequest(httpParameters.getMethod(), uri);
- } else {
- request = createRequest(httpParameters.getMethod(), uri, reqEntity);
- }
-
- if (reqHeaders != null) {
- for (Entry<String, String> entry : reqHeaders.entrySet()) {
- if (entry.getValue() == null || isAscii(entry.getValue())) {
- request.addHeader(entry.getKey(), entry.getValue());
- } else {
- byte[] asBytes = Utf8.toBytes(entry.getValue());
- String asLyingString = new String(asBytes, 0, asBytes.length, iso8859Charset);
- request.addHeader(entry.getKey(), asLyingString);
- }
- }
- }
-
- // Logging
- if (requestMeta != null) {
- for (HeaderIterator headers = request.headerIterator(); headers.hasNext();) {
- Header h = headers.nextHeader();
- requestMeta.setField(LOG_HEADER_PREFIX + h.getName(), h.getValue());
- }
- start = System.currentTimeMillis();
- }
-
- HttpResponse response;
-
- try {
- HttpContext context = new BasicHttpContext();
- response = httpClient.execute(request, context);
-
- if (requestMeta != null) {
- requestMeta.setField(LOG_IP_ADDRESS, getIpAddress(context));
- }
- } catch (ConnectTimeoutException e) {
- connectTimeouts.increment();
- throw e;
- }
-
- // Logging
- long latencyStart = System.currentTimeMillis() - start;
- if (requestMeta != null) {
- requestMeta.setField(LOG_LATENCY_START, latencyStart);
- }
- logResponseLatency(latencyStart);
- return response;
- }
-
- private String getIpAddress(HttpContext context) {
- HttpConnection connection = (HttpConnection) context.getAttribute(ExecutionContext.HTTP_CONNECTION);
- if (connection instanceof HttpInetConnection) {
- InetAddress address = ((HttpInetConnection) connection).getRemoteAddress();
- String hostAddress = address.getHostAddress();
- return hostAddress == null ?
- IP_ADDRESS_UNKNOWN:
- hostAddress;
- } else {
- getLogger().log(LogLevel.DEBUG, "Unexpected connection type: " + connection.getClass().getName());
- return IP_ADDRESS_UNKNOWN;
- }
- }
-
- private boolean isAscii(String value) {
- char[] scanBuffer = new char[value.length()];
- value.getChars(0, value.length(), scanBuffer, 0);
- for (char c: scanBuffer)
- if (c > 127) return false;
- return true;
- }
-
- protected void logResponseLatency(long latency) { }
-
- /**
- * Creates a http client for one request. Override to customize the client
- * to use, e.g for testing. This default implementation will add a certificate store
- * proxy to params if is necessary, and then do
- * <code>return new SearcherHttpClient(getConnectionManager(params), params);</code>
- */
- protected HttpClient createClient(HttpParams params) {
- if (certificateProxy != null) {
- params.setParameter(ConnRoutePNames.DEFAULT_PROXY, certificateProxy);
- }
- return new SearcherHttpClient(getConnectionManager(params), params);
- }
-
- /**
- * Creates a HttpRequest. Override to customize the request.
- * This default implementation does <code>return new HttpRequest(method,uri);</code>
- */
- protected HttpUriRequest createRequest(String method,URI uri) {
- return createRequest(method, uri, null);
- }
-
- /**
- * Creates a HttpRequest. Override to customize the request.
- * This default implementation does <code>return new HttpRequest(method,uri);</code>
- */
- protected HttpUriRequest createRequest(String method,URI uri, HttpEntity entity) {
- return new SearcherHttpRequest(method,uri);
- }
-
- /** Get a connection manager which may be used safely from this thread */
- protected ClientConnectionManager getConnectionManager(HttpParams params) {
- if (sharedConnectionManager != null) {// We are using shared connections
- return sharedConnectionManager;
- } else {
- SingleClientConnManager singleClientConnManager = singleClientConnManagerThreadLocal.get();
- if (singleClientConnManager == null) {
- singleClientConnManager = new SingleClientConnManager(params, schemeRegistry);
- singleClientConnManagerThreadLocal.set(singleClientConnManager);
- }
- return singleClientConnManager;
- }
- }
-
- /** Utility method for creating error messages when a url is incorrect */
- protected ErrorMessage createMalformedUrlError(Query query,Exception e) {
- return ErrorMessage.createErrorInPluginSearcher("Malformed url in " + this + " for " + query +
- ": " + Exceptions.toMessageString(e));
- }
-
- private Map<String, String> generateYCAHeaders() {
- Map<String, String> headers = new HashMap<>();
- String certificate = certificateStore.getCertificate(certificateApplicationId, certificateTtl, certificateRetry);
- headers.put(YCA_HTTP_HEADER, certificate);
- return headers;
- }
-
- protected static class SearcherHttpClient extends DefaultHttpClient {
-
- private final int retries;
-
- public SearcherHttpClient(final ClientConnectionManager conman, final HttpParams params) {
- super(conman, params);
- retries = params.getIntParameter(HTTPParameters.RETRIES, 1);
- addRequestInterceptor((request, context) -> {
- if (!request.containsHeader("Accept-Encoding")) {
- request.addHeader("Accept-Encoding", "gzip");
- }
- });
- addResponseInterceptor((response, context) -> {
- HttpEntity entity = response.getEntity();
- if (entity == null) return;
- Header ceheader = entity.getContentEncoding();
- if (ceheader == null) return;
- for (HeaderElement codec : ceheader.getElements()) {
- if (codec.getName().equalsIgnoreCase("gzip")) {
- response.setEntity(new GzipDecompressingEntity(response.getEntity()));
- return;
- }
- }
- });
- }
-
- @Override
- protected HttpRequestExecutor createRequestExecutor() {
- return new HttpRequestExecutor();
- }
-
- @Override
- protected HttpRoutePlanner createHttpRoutePlanner() {
- return new DefaultHttpRoutePlanner(getConnectionManager().getSchemeRegistry());
- }
-
- @Override
- protected HttpRequestRetryHandler createHttpRequestRetryHandler() {
- return new SearcherHttpRequestRetryHandler(retries);
- }
- }
-
- /** A retry handler which avoids retrying forever on errors misclassified as transient */
- private static class SearcherHttpRequestRetryHandler implements HttpRequestRetryHandler {
- private final int retries;
-
- public SearcherHttpRequestRetryHandler(int retries) {
- this.retries = retries;
- }
-
- @Override
- public boolean retryRequest(IOException e, int executionCount, HttpContext httpContext) {
- if (e == null) {
- throw new IllegalArgumentException("Exception parameter may not be null");
- }
- if (executionCount > retries) {
- return false;
- }
- if (e instanceof NoHttpResponseException) {
- // Retry if the server dropped connection on us
- return true;
- }
- if (e instanceof InterruptedIOException) {
- // Timeout from federation layer
- return false;
- }
- if (e instanceof UnknownHostException) {
- // Unknown host
- return false;
- }
- if (e instanceof SSLHandshakeException) {
- // SSL handshake exception
- return false;
- }
- return true;
- }
-
-
- }
-
- private static class SearcherHttpRequest extends HttpRequestBase {
- String method;
-
- public SearcherHttpRequest(String method, final URI uri) {
- super();
- this.method = method;
- setURI(uri);
- }
-
- @Override
- public String getMethod() {
- return method;
- }
- }
-
- /**
- * Only for testing.
- */
- public void shutdownConnectionManagers() {
- ClientConnectionManager manager;
- if (sharedConnectionManager != null) {
- manager = sharedConnectionManager;
- } else {
- manager = singleClientConnManagerThreadLocal.get();
- }
- if (manager != null) {
- manager.shutdown();
- }
- }
-
- protected static final class ThrowingCertificateStore implements CertificateStore {
-
- @Override
- public String getCertificate(String key, long ttl, long retry) {
- throw new UnsupportedOperationException("A certificate store is not available");
- }
-
- }
-
-}
-
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java
deleted file mode 100644
index 03ffe2b8a9c..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedHttpEntity.java
+++ /dev/null
@@ -1,91 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-
-/**
- * Wrapper for adding timeout to an HttpEntity instance.
- *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class TimedHttpEntity implements HttpEntity {
- /**
- * The wrapped entity. Never null.
- */
- private final HttpEntity entity;
- private final long startTime;
- private final long timeout;
-
- public TimedHttpEntity(HttpEntity entity, long startTime, long timeout) {
- if (entity == null) {
- throw new IllegalArgumentException("TimedHttpEntity cannot be instantiated with null HttpEntity.");
- }
- this.entity = entity;
- this.startTime = startTime;
- this.timeout = timeout;
- }
-
-
- @Override
- public InputStream getContent() throws IOException, IllegalStateException {
- InputStream content = entity.getContent();
- if (content == null) {
- return null;
- } else {
- return new TimedStream(content, startTime, timeout);
- }
- }
-
-
- // START OF PURE FORWARDING METHODS
- @Override
- public void consumeContent() throws IOException {
- entity.consumeContent();
- }
-
-
- @Override
- public Header getContentEncoding() {
- return entity.getContentEncoding();
- }
-
- @Override
- public long getContentLength() {
- return entity.getContentLength();
- }
-
- @Override
- public Header getContentType() {
- return entity.getContentType();
- }
-
- @Override
- public boolean isChunked() {
- return entity.isChunked();
- }
-
- @Override
- public boolean isRepeatable() {
- return entity.isRepeatable();
- }
-
- @Override
- public boolean isStreaming() {
- return entity.isStreaming();
- }
-
- @Override
- public void writeTo(OutputStream outstream) throws IOException {
- entity.writeTo(outstream);
- }
- // END OF PURE FORWARDING METHODS
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java
deleted file mode 100644
index 77a42ee0a34..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/TimedStream.java
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * A stream which throws a TimeoutException if query timeout has been reached.
- *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class TimedStream extends InputStream {
-
- /**
- * A time barrier value, the point in time from which on read operations will cause an exception.
- */
- private final long limit;
-
- /**
- * A wrapped InputStream instance.
- */
- private final InputStream content;
-
- /**
- * Wrap an InputStream to make read operations potentially fire off
- * TimeoutException.
- *
- * <p>Typical use would be<br>
- * <code>new TimedStream(httpEntity.getContent(), query.getStartTime(), query.getTimeout())</code>
- *
- * @param content
- * the InputStream to wrap
- * @param startTime
- * start time of query
- * @param timeout
- * how long the query is allowed to run
- */
- public TimedStream(InputStream content, long startTime, long timeout) {
- if (content == null) {
- throw new IllegalArgumentException("Cannot instantiate TimedStream with null InputStream");
- }
- this.content = content;
- // The reasion for doing it in here instead of outside the constructor
- // is this makes the usage of the class more intuitive IMHO
- this.limit = startTime + timeout;
- }
-
- private void checkTime(String message) {
- if (System.currentTimeMillis() >= limit) {
- throw new TimeoutException(message);
- }
- }
-
- // START FORWARDING METHODS:
- // All methods below are forwarding methods to the contained stream, where
- // some do a timeout check.
- @Override
- public int read() throws IOException {
- int data = content.read();
- checkTime("Timed out during read().");
- return data;
- }
-
- @Override
- public int available() throws IOException {
- return content.available();
- }
-
- @Override
- public void close() throws IOException {
- content.close();
- }
-
- @Override
- public synchronized void mark(int readlimit) {
- content.mark(readlimit);
- }
-
- @Override
- public boolean markSupported() {
- return content.markSupported();
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- int length = content.read(b, off, len);
- checkTime("Timed out during read(byte[], int, int)");
- return length;
- }
-
- @Override
- public int read(byte[] b) throws IOException {
- int length = content.read(b);
- checkTime("Timed out during read(byte[])");
- return length;
- }
-
- @Override
- public synchronized void reset() throws IOException {
- content.reset();
- }
-
- @Override
- public long skip(long n) throws IOException {
- long skipped = content.skip(n);
- checkTime("Timed out during skip(long)");
- return skipped;
- }
- // END FORWARDING METHODS
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java b/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java
deleted file mode 100644
index 18dd59ec37b..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/TimeoutException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-/**
- * Timeout marker for slow HTTP connections.
- *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class TimeoutException extends RuntimeException {
-
- /**
- * Auto-generated version ID.
- */
- private static final long serialVersionUID = 7084147598258586559L;
-
- public TimeoutException(String message) {
- super(message);
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java
deleted file mode 100644
index af334b35221..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/http/package-info.java
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-@ExportPackage
-package com.yahoo.search.federation.http;
-
-import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java
deleted file mode 100644
index 3a0db9b76de..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/vespa/QueryMarshaller.java
+++ /dev/null
@@ -1,176 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.vespa;
-
-import java.util.Iterator;
-
-import com.yahoo.prelude.query.*;
-
-/**
- * Marshal a query stack into an advanced query string suitable for
- * passing to another QRS.
- *
- * @author Steinar Knutsen
- * @author Rong-En Fan
- * @deprecated use YQL
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-public class QueryMarshaller {
-
- private boolean atRoot = true;
-
- public String marshal(Item root) {
- if (root == null || root instanceof NullItem) {
- return null;
- }
- StringBuilder s = new StringBuilder();
- marshal(root, s);
- atRoot = true;
- return s.toString();
- }
-
- /**
- * We do not yet care about exact match indices
- */
- private void marshal(Item root, StringBuilder s) {
- switch (root.getItemType()) {
- case OR:
- marshalOr((OrItem) root, s);
- break;
- case AND:
- marshalAnd((CompositeItem) root, s);
- break;
- case NOT:
- marshalNot((NotItem) root, s);
- break;
- case RANK:
- marshalRank((RankItem) root, s);
- break;
- case WORD:
- case INT:
- case PREFIX:
- case SUBSTRING:
- case SUFFIX:
- marshalWord((TermItem) root, s);
- break;
- case PHRASE:
- // PhraseItem and PhraseSegmentItem don't add quotes for segmented
- // termse
- if (root instanceof PhraseSegmentItem) {
- marshalPhrase((PhraseSegmentItem) root, s);
- } else {
- marshalPhrase((PhraseItem) root, s);
- }
- break;
- case NEAR:
- marshalNear((NearItem) root, s);
- break;
- case ONEAR:
- marshalNear((ONearItem) root, s);
- break;
- case WEAK_AND:
- marshalWeakAnd((WeakAndItem)root, s);
- break;
- default:
- break;
- }
- }
-
-
- private void marshalWord(TermItem item, StringBuilder s) {
- String index = item.getIndexName();
- if (index.length() != 0) {
- s.append(item.getIndexName()).append(':');
- }
- s.append(item.stringValue());
- if (item.getWeight() != Item.DEFAULT_WEIGHT)
- s.append("!").append(item.getWeight());
- }
-
- private void marshalRank(RankItem root, StringBuilder s) {
- marshalComposite("RANK", root, s);
- }
-
- private void marshalNot(NotItem root, StringBuilder s) {
- marshalComposite("ANDNOT", root, s);
- }
-
- private void marshalOr(OrItem root, StringBuilder s) {
- marshalComposite("OR", root, s);
- }
-
- /**
- * Dump WORD items, and add space between each of them unless those
- * words came from segmentation.
- *
- * @param root CompositeItem
- * @param s current marshaled query
- */
- private void dumpWords(CompositeItem root, StringBuilder s) {
- for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) {
- Item word = i.next();
- boolean useSeparator = true;
- if (word instanceof TermItem) {
- s.append(((TermItem) word).stringValue());
- if (word instanceof WordItem) {
- useSeparator = !((WordItem) word).isFromSegmented();
- }
- } else {
- dumpWords((CompositeItem) word, s);
- }
- if (useSeparator && i.hasNext()) {
- s.append(' ');
- }
- }
- }
-
- private void marshalPhrase(PhraseItem root, StringBuilder s) {
- marshalPhrase(root, s, root.isExplicit(), false);
- }
-
- private void marshalPhrase(PhraseSegmentItem root, StringBuilder s) {
- marshalPhrase(root, s, root.isExplicit(), true);
- }
-
- private void marshalPhrase(IndexedItem root, StringBuilder s, boolean isExplicit, boolean isSegmented) {
- String index = root.getIndexName();
- if (index.length() != 0) {
- s.append(root.getIndexName()).append(':');
- }
- if (isExplicit || !isSegmented) s.append('"');
- dumpWords((CompositeItem) root, s);
- if (isExplicit || !isSegmented) s.append('"');
- }
-
- private void marshalNear(NearItem root, StringBuilder s) {
- marshalComposite(root.getName() + "(" + root.getDistance() + ")", root, s);
- }
-
- // Not only AndItem returns ItemType.AND
- private void marshalAnd(CompositeItem root, StringBuilder s) {
- marshalComposite("AND", root, s);
- }
-
- private void marshalWeakAnd(WeakAndItem root, StringBuilder s) {
- marshalComposite("WAND(" + root.getN() + ")", root, s);
- }
-
- private void marshalComposite(String operator, CompositeItem root, StringBuilder s) {
- boolean useParen = !atRoot;
- if (useParen) {
- s.append("( ");
- } else {
- atRoot = false;
- }
- for (Iterator<Item> i = root.getItemIterator(); i.hasNext();) {
- Item item = i.next();
- marshal(item, s);
- if (i.hasNext())
- s.append(' ').append(operator).append(' ');
- }
- if (useParen) {
- s.append(" )");
- }
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java
deleted file mode 100644
index a6aec30b496..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/vespa/ResultBuilder.java
+++ /dev/null
@@ -1,642 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.vespa;
-
-import com.yahoo.log.LogLevel;
-import com.yahoo.prelude.hitfield.XMLString;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.result.ErrorMessage;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.result.HitGroup;
-import com.yahoo.search.result.Relevance;
-import com.yahoo.text.XML;
-import org.xml.sax.*;
-import org.xml.sax.helpers.DefaultHandler;
-import org.xml.sax.helpers.XMLReaderFactory;
-
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Logger;
-
-import static com.yahoo.text.Lowercase.toLowerCase;
-
-/**
- * Parse Vespa XML results and create Result instances.
- *
- * @author Steinar Knutsen
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated // OK
-@SuppressWarnings("deprecation")
-public class ResultBuilder extends DefaultHandler {
- private static final String ERROR = "error";
-
- private static final String FIELD = "field";
-
- private static Logger log = Logger.getLogger(ResultBuilder.class.getName());
-
- /** Namespaces feature id (http://xml.org/sax/features/namespaces). */
- protected static final String NAMESPACES_FEATURE_ID = "http://xml.org/sax/features/namespaces";
-
- /**
- * Namespace prefixes feature id
- * (http://xml.org/sax/features/namespace-prefixes).
- */
- protected static final String NAMESPACE_PREFIXES_FEATURE_ID = "http://xml.org/sax/features/namespace-prefixes";
-
- /** Validation feature id (http://xml.org/sax/features/validation). */
- protected static final String VALIDATION_FEATURE_ID = "http://xml.org/sax/features/validation";
-
- /**
- * Schema validation feature id
- * (http://apache.org/xml/features/validation/schema).
- */
- protected static final String SCHEMA_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/schema";
-
- /**
- * Dynamic validation feature id
- * (http://apache.org/xml/features/validation/dynamic).
- */
- protected static final String DYNAMIC_VALIDATION_FEATURE_ID = "http://apache.org/xml/features/validation/dynamic";
-
- // default settings
-
- /** Default parser name. */
- protected static final String DEFAULT_PARSER_NAME = "org.apache.xerces.parsers.SAXParser";
-
- /** Default namespaces support (false). */
- protected static final boolean DEFAULT_NAMESPACES = false;
-
- /** Default namespace prefixes (false). */
- protected static final boolean DEFAULT_NAMESPACE_PREFIXES = false;
-
- /** Default validation support (false). */
- protected static final boolean DEFAULT_VALIDATION = false;
-
- /** Default Schema validation support (false). */
- protected static final boolean DEFAULT_SCHEMA_VALIDATION = false;
-
- /** Default dynamic validation support (false). */
- protected static final boolean DEFAULT_DYNAMIC_VALIDATION = false;
-
- private StringBuilder fieldContent;
-
- private String fieldName;
-
- private int fieldLevel = 0;
-
- private boolean hasLiteralTags = false;
-
- private Map<String, Object> hitFields = new HashMap<>();
- private String hitType;
- private String hitRelevance;
- private String hitSource;
-
- private int offset = 0;
-
- private List<Tag> tagStack = new ArrayList<>();
-
- private final XMLReader parser;
-
- private Query query;
-
- private Result result;
-
- private static enum ResultPart {
- ROOT, ERRORDETAILS, HIT, HITGROUP;
- }
-
- Deque<ResultPart> location = new ArrayDeque<>(10);
-
- private String currentErrorCode;
-
- private String currentError;
-
- private Deque<HitGroup> hitGroups = new ArrayDeque<>(5);
-
- private static class Tag {
- public final String name;
-
- /**
- * Offset is a number which is generated for all data and tags inside
- * fields, used to determine whether a tag was closed without enclosing
- * any characters or other tags.
- */
- public final int offset;
-
- public Tag(final String name, final int offset) {
- this.name = name;
- this.offset = offset;
- }
-
- @Override
- public String toString() {
- return name + '(' + Integer.valueOf(offset) + ')';
- }
- }
-
- /** Default constructor. */
- public ResultBuilder() throws RuntimeException {
- this(createParser());
- }
-
- public ResultBuilder(XMLReader parser) {
- this.parser = parser;
- this.parser.setContentHandler(this);
- this.parser.setErrorHandler(this);
- }
-
- public static XMLReader createParser() {
- ClassLoader savedContextClassLoader = Thread.currentThread().getContextClassLoader();
- Thread.currentThread().setContextClassLoader(org.apache.xerces.parsers.SAXParser.class.getClassLoader());
-
- try {
- XMLReader reader = XMLReaderFactory.createXMLReader(DEFAULT_PARSER_NAME);
- setParserFeatures(reader);
- return reader;
- } catch (Exception e) {
- throw new RuntimeException("error: Unable to instantiate parser ("
- + DEFAULT_PARSER_NAME + ")", e);
- } finally {
- Thread.currentThread().setContextClassLoader(savedContextClassLoader);
- }
- }
-
- private static void setParserFeatures(XMLReader reader) {
- try {
- reader.setFeature(NAMESPACES_FEATURE_ID, DEFAULT_NAMESPACES);
- } catch (SAXException e) {
- log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
- + NAMESPACES_FEATURE_ID + ")");
- }
- try {
- reader.setFeature(NAMESPACE_PREFIXES_FEATURE_ID,
- DEFAULT_NAMESPACE_PREFIXES);
- } catch (SAXException e) {
- log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
- + NAMESPACE_PREFIXES_FEATURE_ID + ")");
- }
- try {
- reader.setFeature(VALIDATION_FEATURE_ID, DEFAULT_VALIDATION);
- } catch (SAXException e) {
- log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
- + VALIDATION_FEATURE_ID + ")");
- }
- try {
- reader.setFeature(SCHEMA_VALIDATION_FEATURE_ID,
- DEFAULT_SCHEMA_VALIDATION);
- } catch (SAXNotRecognizedException e) {
- log.log(LogLevel.WARNING, "warning: Parser does not recognize feature ("
- + SCHEMA_VALIDATION_FEATURE_ID + ")");
-
- } catch (SAXNotSupportedException e) {
- log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
- + SCHEMA_VALIDATION_FEATURE_ID + ")");
- }
-
- try {
- reader.setFeature(DYNAMIC_VALIDATION_FEATURE_ID,
- DEFAULT_DYNAMIC_VALIDATION);
- } catch (SAXNotRecognizedException e) {
- log.log(LogLevel.WARNING, "warning: Parser does not recognize feature ("
- + DYNAMIC_VALIDATION_FEATURE_ID + ")");
-
- } catch (SAXNotSupportedException e) {
- log.log(LogLevel.WARNING, "warning: Parser does not support feature ("
- + DYNAMIC_VALIDATION_FEATURE_ID + ")");
- }
- }
-
- @Override
- public void startDocument() throws SAXException {
- reset();
- result = new Result(query);
- hitGroups.addFirst(result.hits());
- location.addFirst(ResultPart.ROOT);
- return;
- }
-
- private void reset() {
- result = null;
- fieldLevel = 0;
- hasLiteralTags = false;
- tagStack = null;
- fieldContent = null;
- offset = 0;
- currentError = null;
- currentErrorCode = null;
- hitGroups.clear();
- location.clear();
- }
-
- @Override
- public void startElement(String uri, String local, String raw,
- Attributes attrs) throws SAXException {
- // "Everybody" wants this switch to be moved into the
- // enum class instead, but in this case, I find the classic
- // approach more readable.
- switch (location.peekFirst()) {
- case HIT:
- if (fieldLevel > 0) {
- tagInField(raw, attrs, FIELD);
- ++offset;
- return;
- }
- if (FIELD.equals(raw)) {
- ++fieldLevel;
- fieldName = attrs.getValue("name");
- fieldContent = new StringBuilder();
- hasLiteralTags = false;
- }
- break;
- case ERRORDETAILS:
- if (fieldLevel > 0) {
- tagInField(raw, attrs, ERROR);
- ++offset;
- return;
- }
- if (ERROR.equals(raw)) {
- if (attrs != null) {
- currentErrorCode = attrs.getValue("code");
- currentError = attrs.getValue("error");
- }
- ++fieldLevel;
- fieldContent = new StringBuilder();
- hasLiteralTags = false;
- }
- break;
- case HITGROUP:
- if ("hit".equals(raw)) {
- startHit(attrs);
- } else if ("group".equals(raw)) {
- startHitGroup(attrs);
- }
- break;
- case ROOT:
- if ("hit".equals(raw)) {
- startHit(attrs);
- } else if ("errordetails".equals(raw)) {
- location.addFirst(ResultPart.ERRORDETAILS);
- } else if ("result".equals(raw)) {
- if (attrs != null) {
- String total = attrs.getValue("total-hit-count");
- if (total != null) {
- result.setTotalHitCount(Long.valueOf(total));
- }
- }
- } else if ("group".equals(raw)) {
- startHitGroup(attrs);
- } else if (ERROR.equals(raw)) {
- if (attrs != null) {
- currentErrorCode = attrs.getValue("code");
- fieldContent = new StringBuilder();
- }
- }
- break;
- }
- ++offset;
- }
-
- private void startHitGroup(Attributes attrs) {
- HitGroup g = new HitGroup();
- Set<String> types = g.types();
-
- final String source;
- if (attrs != null) {
- String groupType = attrs.getValue("type");
- if (groupType != null) {
- for (String s : groupType.split(" ")) {
- if (s.length() > 0) {
- types.add(s);
- }
- }
- }
-
- source = attrs.getValue("source");
- } else {
- source = null;
- }
-
- g.setId((source != null) ? source : "dummy");
-
- hitGroups.peekFirst().add(g);
- hitGroups.addFirst(g);
- location.addFirst(ResultPart.HITGROUP);
- }
-
- private void startHit(Attributes attrs) {
- hitFields.clear();
- location.addFirst(ResultPart.HIT);
- if (attrs != null) {
- hitRelevance = attrs.getValue("relevancy");
- hitSource = attrs.getValue("source");
- hitType = attrs.getValue("type");
- } else {
- hitRelevance = null;
- hitSource = null;
- hitType = null;
- }
- }
-
- private void tagInField(String tag, Attributes attrs, String enclosingTag) {
- if (!hasLiteralTags) {
- hasLiteralTags = true;
- String fieldTillNow = XML.xmlEscape(fieldContent.toString(), false);
- fieldContent = new StringBuilder(fieldTillNow);
- tagStack = new ArrayList<>();
- }
- if (enclosingTag.equals(tag)) {
- ++fieldLevel;
- }
- if (tagStack.size() > 0) {
- Tag prevTag = tagStack.get(tagStack.size() - 1);
- if (prevTag != null && (prevTag.offset + 1) == offset) {
- fieldContent.append(">");
- }
- }
- fieldContent.append("<").append(tag);
- if (attrs != null) {
- int attrCount = attrs.getLength();
- for (int i = 0; i < attrCount; i++) {
- fieldContent.append(" ").append(attrs.getQName(i))
- .append("=\"").append(
- XML.xmlEscape(attrs.getValue(i), true)).append(
- "\"");
- }
- }
- tagStack.add(new Tag(tag, offset));
- }
-
- private void endElementInField(String qName, String enclosingTag) {
- Tag prevTag = tagStack.get(tagStack.size() - 1);
- if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) {
- fieldContent.append(" />");
- } else {
- fieldContent.append("</").append(qName).append('>');
- }
- if (prevTag.name.equals(qName)) {
- tagStack.remove(tagStack.size() - 1);
- }
- }
-
- private void endElementInHitField(String qName) {
- if (FIELD.equals(qName) && --fieldLevel == 0) {
- Object content;
- if (hasLiteralTags) {
- content = new XMLString(fieldContent.toString());
- } else {
- content = fieldContent.toString();
- }
- hitFields.put(fieldName, content);
- if ("collapseId".equals(fieldName)) {
- hitFields.put(fieldName, Integer.valueOf(content.toString()));
- }
- fieldName = null;
- fieldContent = null;
- tagStack = null;
- } else {
- Tag prevTag = tagStack.get(tagStack.size() - 1);
- if (qName.equals(prevTag.name) && offset == (prevTag.offset + 1)) {
- fieldContent.append(" />");
- } else {
- fieldContent.append("</").append(qName).append('>');
- }
- if (prevTag.name.equals(qName)) {
- tagStack.remove(tagStack.size() - 1);
- }
- }
- }
- @Override
- public void characters(char ch[], int start, int length)
- throws SAXException {
-
- switch (location.peekFirst()) {
- case ERRORDETAILS:
- case HIT:
- if (fieldLevel > 0) {
- if (hasLiteralTags) {
- if (tagStack.size() > 0) {
- Tag tag = tagStack.get(tagStack.size() - 1);
- if (tag != null && (tag.offset + 1) == offset) {
- fieldContent.append(">");
- }
- }
- fieldContent.append(
- XML.xmlEscape(new String(ch, start, length), false));
- } else {
- fieldContent.append(ch, start, length);
- }
- }
- break;
- default:
- if (fieldContent != null) {
- fieldContent.append(ch, start, length);
- }
- break;
- }
- ++offset;
- }
-
- @Override
- public void ignorableWhitespace(char ch[], int start, int length)
- throws SAXException {
- return;
- }
-
- @Override
- public void processingInstruction(String target, String data)
- throws SAXException {
- return;
- }
-
- @Override
- public void endElement(String namespaceURI, String localName, String qName)
- throws SAXException {
- switch (location.peekFirst()) {
- case HITGROUP:
- if ("group".equals(qName)) {
- hitGroups.removeFirst();
- location.removeFirst();
- }
- break;
- case HIT:
- if (fieldLevel > 0) {
- endElementInHitField(qName);
- } else if ("hit".equals(qName)) {
- //assert(hitKeys.size() == hitValues.size());
- //We try to get either uri or documentID and use that as id
- Object docId = extractDocumentID();
- Hit newHit = new Hit(docId.toString());
- if (hitRelevance != null) newHit.setRelevance(new Relevance(Double.parseDouble(hitRelevance)));
- if(hitSource != null) newHit.setSource(hitSource);
- if(hitType != null) {
- for(String type: hitType.split(" ")) {
- newHit.types().add(type);
- }
- }
- for(Map.Entry<String, Object> field : hitFields.entrySet()) {
- newHit.setField(field.getKey(), field.getValue());
- }
-
- hitGroups.peekFirst().add(newHit);
- location.removeFirst();
- }
- break;
- case ERRORDETAILS:
- if (fieldLevel == 1 && ERROR.equals(qName)) {
- ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode),
- currentError,
- fieldContent.toString());
- hitGroups.peekFirst().addError(error);
- currentError = null;
- currentErrorCode = null;
- fieldContent = null;
- tagStack = null;
- fieldLevel = 0;
- } else if (fieldLevel > 0) {
- endElementInField(qName, ERROR);
- } else if ("errordetails".equals(qName)) {
- location.removeFirst();
- }
- break;
- case ROOT:
- if (ERROR.equals(qName)) {
- ErrorMessage error = new ErrorMessage(Integer.valueOf(currentErrorCode),
- fieldContent.toString());
- hitGroups.peekFirst().addError(error);
- currentErrorCode = null;
- fieldContent = null;
- }
- break;
- default:
- break;
- }
- ++offset;
- }
-
- private Object extractDocumentID() {
- Object docId = null;
- if (hitFields.containsKey("uri")) {
- docId = hitFields.get("uri");
- } else {
- final String documentId = "documentId";
- if (hitFields.containsKey(documentId)) {
- docId = hitFields.get(documentId);
- } else {
- final String lcDocumentId = toLowerCase(documentId);
- for (Map.Entry<String, Object> e : hitFields.entrySet()) {
- String key = e.getKey();
- // case insensitive matching, checking length first hoping to avoid some lowercasing
- if (documentId.length() == key.length() && lcDocumentId.equals(toLowerCase(key))) {
- docId = e.getValue();
- break;
- }
- }
- }
- }
- if (docId == null) {
- docId = "dummy";
- log.info("Results from vespa backend did not contain either uri or documentId");
- }
- return docId;
- }
-
- @Override
- public void warning(SAXParseException ex) throws SAXException {
- printError("Warning", ex);
- }
-
- @Override
- public void error(SAXParseException ex) throws SAXException {
- printError("Error", ex);
- }
-
- @Override
- public void fatalError(SAXParseException ex) throws SAXException {
- printError("Fatal Error", ex);
- // throw ex;
- }
-
- /** Prints the error message. */
- protected void printError(String type, SAXParseException ex) {
- StringBuilder errorMessage = new StringBuilder();
-
- errorMessage.append(type);
- if (ex != null) {
- String systemId = ex.getSystemId();
- if (systemId != null) {
- int index = systemId.lastIndexOf('/');
- if (index != -1)
- systemId = systemId.substring(index + 1);
- errorMessage.append(' ').append(systemId);
- }
- }
- errorMessage.append(':')
- .append(ex.getLineNumber())
- .append(':')
- .append(ex.getColumnNumber())
- .append(": ")
- .append(ex.getMessage());
- log.log(LogLevel.WARNING, errorMessage.toString());
-
- }
-
- public Result parse(String identifier, Query query) {
- Result toReturn;
-
- setQuery(query);
- try {
- parser.parse(identifier);
- } catch (SAXParseException e) {
- // ignore
- } catch (Exception e) {
- log.log(LogLevel.WARNING, "Error parsing result from Vespa",e);
- Exception se = e;
- if (e instanceof SAXException) {
- se = ((SAXException) e).getException();
- }
- if (se != null)
- se.printStackTrace(System.err);
- else
- e.printStackTrace(System.err);
- }
- toReturn = result;
- reset();
- return toReturn;
- }
-
- public Result parse(InputSource input, Query query) {
- Result toReturn;
-
- setQuery(query);
- try {
- parser.parse(input);
- } catch (SAXParseException e) {
- // ignore
- } catch (Exception e) {
- log.log(LogLevel.WARNING, "Error parsing result from Vespa",e);
- Exception se = e;
- if (e instanceof SAXException) {
- se = ((SAXException) e).getException();
- }
- if (se != null)
- se.printStackTrace(System.err);
- else
- e.printStackTrace(System.err);
- }
- toReturn = result;
- reset();
- return toReturn;
- }
-
-
- private void setQuery(Query query) {
- this.query = query;
- }
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
deleted file mode 100644
index 246732d0970..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/vespa/VespaSearcher.java
+++ /dev/null
@@ -1,259 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.vespa;
-
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.Map;
-import java.util.Set;
-
-import com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher;
-import org.xml.sax.InputSource;
-import org.xml.sax.XMLReader;
-
-import com.google.inject.Inject;
-import com.yahoo.collections.Tuple2;
-import com.yahoo.component.ComponentId;
-import com.yahoo.component.Version;
-import com.yahoo.component.chain.dependencies.After;
-import com.yahoo.component.chain.dependencies.Provides;
-import com.yahoo.language.Linguistics;
-import com.yahoo.log.LogLevel;
-import com.yahoo.prelude.query.Item;
-import com.yahoo.prelude.query.QueryCanonicalizer;
-import com.yahoo.processing.request.CompoundName;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.cache.QrBinaryCacheConfig;
-import com.yahoo.search.cache.QrBinaryCacheRegionConfig;
-import com.yahoo.search.federation.ProviderConfig;
-import com.yahoo.search.federation.http.Connection;
-import com.yahoo.search.query.QueryTree;
-import com.yahoo.search.query.textserialize.TextSerialize;
-import com.yahoo.search.yql.MinimalQueryInserter;
-import com.yahoo.statistics.Statistics;
-
-import edu.umd.cs.findbugs.annotations.Nullable;
-
-/**
- * Backend searcher for external Vespa clusters (queried over http).
- *
- * <p>If the "sources" argument should be honored on an external cluster
- * when using YQL+, override {@link #chooseYqlSources(Set)}.</p>
- *
- * @author Arne Bergene Fossaa
- * @author Steinar Knutsen
- * @deprecated
- */
-// TODO: Remove on Vespa 7
-@Deprecated
-@Provides("Vespa")
-@After("*")
-public class VespaSearcher extends ConfiguredHTTPProviderSearcher {
-
- private final ThreadLocal<XMLReader> readerHolder = new ThreadLocal<>();
- private final Query.Type queryType;
- private final Tuple2<String, Version> segmenterVersion;
-
- private static final CompoundName select = new CompoundName("select");
- private static final CompoundName streamingUserid = new CompoundName("streaming.userid");
- private static final CompoundName streamingGroupname = new CompoundName("streaming.groupname");
- private static final CompoundName streamingSelection = new CompoundName("streaming.selection");
-
- /** Create an instance from configuration */
- public VespaSearcher(ComponentId id, ProviderConfig config, QrBinaryCacheConfig c,
- QrBinaryCacheRegionConfig r, Statistics statistics) {
- this(id, config, c, r, statistics, null);
- }
-
- /**
- * Create an instance from configuration
- *
- * @param linguistics used for generating meta info for YQL+
- */
- @Inject
- public VespaSearcher(ComponentId id, ProviderConfig config,
- QrBinaryCacheConfig c, QrBinaryCacheRegionConfig r,
- Statistics statistics, @Nullable Linguistics linguistics) {
- super(id, config, c, r, statistics);
- queryType = toQueryType(config.queryType());
- if (linguistics == null) {
- segmenterVersion = null;
- } else {
- segmenterVersion = linguistics.getVersion(Linguistics.Component.SEGMENTER);
- }
- }
-
- /**
- * Create an instance from direct parameters having a single connection.
- * Useful for testing
- */
- public VespaSearcher(String idString, String host, int port, String path) {
- super(idString, host, port, path, Statistics.nullImplementation);
- queryType = toQueryType(ProviderConfig.QueryType.LEGACY);
- segmenterVersion = null;
- }
-
- void addProperty(Map<String, String> queryMap, Query query, CompoundName property) {
- Object o = query.properties().get(property);
- if (o != null) {
- queryMap.put(property.toString(), o.toString());
- }
- }
-
- @Override
- public Map<String, String> getQueryMap(Query query) {
- Map<String, String> queryMap = getQueryMapWithoutHitsOffset(query);
- queryMap.put("offset", Integer.toString(query.getOffset()));
- queryMap.put("hits", Integer.toString(query.getHits()));
- queryMap.put("presentation.format", "xml");
-
- addProperty(queryMap, query, select);
- addProperty(queryMap, query, streamingUserid);
- addProperty(queryMap, query, streamingGroupname);
- addProperty(queryMap, query, streamingSelection);
- return queryMap;
- }
-
- @Override
- public Map<String, String> getCacheKey(Query q) {
- return getQueryMapWithoutHitsOffset(q);
- }
-
- private Map<String, String> getQueryMapWithoutHitsOffset(Query query) {
- Map<String, String> queryMap = super.getQueryMap(query);
- if (queryType == Query.Type.YQL) {
- queryMap.put(MinimalQueryInserter.YQL.toString(), marshalQuery(query));
- } else {
- queryMap.put("query", marshalQuery(query.getModel().getQueryTree()));
- queryMap.put("type", queryType.toString());
- }
-
- addNonExcludedSourceProperties(query, queryMap);
- return queryMap;
- }
-
- Query.Type toQueryType(ProviderConfig.QueryType.Enum providerQueryType) {
- if (providerQueryType == ProviderConfig.QueryType.LEGACY) {
- return Query.Type.ADVANCED;
- } else if (providerQueryType == ProviderConfig.QueryType.PROGRAMMATIC) {
- return Query.Type.PROGRAMMATIC;
- } else if (providerQueryType == ProviderConfig.QueryType.YQL) {
- return Query.Type.YQL;
- } else if (providerQueryType == ProviderConfig.QueryType.SELECT) {
- return Query.Type.SELECT;
- } else {
- throw new RuntimeException("Query type " + providerQueryType + " unsupported.");
- }
- }
-
- /**
- * Serialize the query parameter for outgoing queries. For YQL+ queries,
- * sources and fields will be set to all sources and all fields, to follow
- * the behavior of other query types.
- *
- * @param query
- * the current, outgoing query
- * @return a string to include in an HTTP request
- */
- public String marshalQuery(Query query) {
- if (queryType != Query.Type.YQL) {
- return marshalQuery(query.getModel().getQueryTree());
- }
-
- query.getModel().getQueryTree(); // performance: parse query before cloning such that it is only done once
- Query workQuery = query.clone();
- String error = QueryCanonicalizer.canonicalize(workQuery);
- if (error != null) {
- getLogger().log(LogLevel.WARNING,"Could not normalize [" + query.toString() + "]: " + error);
- // Just returning null here is the pattern from existing code...
- return null;
- }
- chooseYqlSources(workQuery.getModel().getSources());
- chooseYqlSummaryFields(workQuery.getPresentation().getSummaryFields());
- return workQuery.yqlRepresentation(getSegmenterVersion(), false);
- }
-
- public String marshalQuery(QueryTree root) {
- QueryTree rootClone = root.clone(); // TODO: Why?
- String error = QueryCanonicalizer.canonicalize(rootClone);
- if (error != null) return null;
-
- return marshalRoot(rootClone.getRoot());
- }
-
- private String marshalRoot(Item root) {
- switch (queryType) {
- case ADVANCED: return new QueryMarshaller().marshal(root);
- case PROGRAMMATIC: return TextSerialize.serialize(root);
- default: throw new RuntimeException("Unsupported query type.");
- }
- }
-
- private XMLReader getReader() {
- XMLReader reader = readerHolder.get();
- if (reader == null) {
- reader = ResultBuilder.createParser();
- readerHolder.set(reader);
- }
- return reader;
- }
-
- @Override
- public void unmarshal(InputStream stream, long contentLength, Result result) {
- ResultBuilder parser = new ResultBuilder(getReader());
- Result mResult = parser.parse(new InputSource(stream),
- result.getQuery());
- result.mergeWith(mResult);
- result.hits().addAll(mResult.hits().asUnorderedHits());
- }
-
- /** Returns the canonical Vespa ping URI, http://host:port/status.html */
- @Override
- public URI getPingURI(Connection connection) throws MalformedURLException, URISyntaxException {
- return new URL(getParameters().getSchema(), connection.getHost(),
- connection.getPort(), "/status.html").toURI();
- }
-
- /**
- * Get the segmenter version data used when creating YQL queries. Useful if
- * overriding {@link #marshalQuery(Query)}.
- *
- * @return a tuple with the name of the segmenting engine in use, and its
- * version
- */
- protected Tuple2<String, Version> getSegmenterVersion() {
- return segmenterVersion;
- }
-
- /**
- * Choose which source arguments to use for the external cluster when
- * generating a YQL+ query string. This is called from
- * {@link #marshalQuery(Query)}. The default implementation clears the set,
- * i.e. requests all sources. Other implementations may modify the source
- * set as they see fit, or simply do nothing.
- *
- * @param sources
- * the set of source names to use for the outgoing query
- */
- protected void chooseYqlSources(Set<String> sources) {
- sources.clear();
- }
-
- /**
- * Choose which summary fields to request from the external cluster when
- * generating a YQL+ query string. This is called from
- * {@link #marshalQuery(Query)}. The default implementation clears the set,
- * i.e. requests all fields. Other implementations may modify the summary
- * field set as they see fit, or simply do nothing.
- *
- * @param summaryFields
- * the set of source names to use for the outgoing query
- */
- protected void chooseYqlSummaryFields(Set<String> summaryFields) {
- summaryFields.clear();
- }
-
-}
diff --git a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java b/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java
deleted file mode 100644
index a1c9236ff9c..00000000000
--- a/container-search/src/main/java/com/yahoo/search/federation/vespa/package-info.java
+++ /dev/null
@@ -1,5 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-@ExportPackage
-package com.yahoo.search.federation.vespa;
-
-import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java
deleted file mode 100644
index a679b17b6fc..00000000000
--- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/HttpProviderSpec.java
+++ /dev/null
@@ -1,118 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.searchchain.model.federation;
-
-import com.yahoo.container.bundle.BundleInstantiationSpecification;
-import net.jcip.annotations.Immutable;
-
-import com.yahoo.search.federation.http.HTTPProviderSearcher;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Specifies how a http provider is to be set up.
- *
- * @author Tony Vaagenes
- * @deprecated
- */
-@Immutable
-@Deprecated // OK
-// TODO: Remove on Vespa 7
-public class HttpProviderSpec {
-
- public enum Type {
- vespa(com.yahoo.search.federation.vespa.VespaSearcher.class);
-
- Type(Class<? extends HTTPProviderSearcher> searcherClass) {
- className = searcherClass.getName();
- }
-
- final String className;
- }
-
- // The default connection parameter values come from the config definition
- public static class ConnectionParameters {
- public final Double readTimeout;
- public final Double connectionTimeout;
- public final Double connectionPoolTimeout;
- public final Integer retries;
-
- public ConnectionParameters(Double readTimeout, Double connectionTimeout,
- Double connectionPoolTimeout, Integer retries) {
- this.readTimeout = readTimeout;
- this.connectionTimeout = connectionTimeout;
- this.connectionPoolTimeout = connectionPoolTimeout;
- this.retries = retries;
- }
- }
-
- public static class Node {
- public final String host;
- public final int port;
-
- public Node(String host, int port) {
- this.host = host;
- this.port = port;
- }
-
- @Override
- public String toString() {
- return "Node{" +
- "host='" + host + '\'' +
- ", port=" + port +
- '}';
- }
- }
-
- public final ConnectionParameters connectionParameters;
-
- public final Integer cacheSizeMB;
-
- public final String path;
- public final List<Node> nodes;
- public final String ycaApplicationId;
- public final Integer ycaCertificateTtl;
- public final Integer ycaRetryWait;
- public final Node ycaProxy;
-
- public static BundleInstantiationSpecification toBundleInstantiationSpecification(Type type) {
- return BundleInstantiationSpecification.getInternalSearcherSpecificationFromStrings(type.className, null);
- }
-
- public static boolean includesType(String typeString) {
- for (Type type : Type.values()) {
- if (type.name().equals(typeString)) {
- return true;
- }
- }
- return false;
- }
-
- public HttpProviderSpec(Double cacheWeight,
- String path,
- List<Node> nodes,
- String ycaApplicationId,
- Integer ycaCertificateTtl,
- Integer ycaRetryWait,
- Node ycaProxy,
- Integer cacheSizeMB,
- ConnectionParameters connectionParameters) {
-
- this.path = path;
- this.nodes = unmodifiable(nodes);
- this.ycaApplicationId = ycaApplicationId;
- this.ycaProxy = ycaProxy;
- this.ycaCertificateTtl = ycaCertificateTtl;
- this.ycaRetryWait = ycaRetryWait;
- this.cacheSizeMB = cacheSizeMB;
-
- this.connectionParameters = connectionParameters;
- }
-
- private List<HttpProviderSpec.Node> unmodifiable(List<HttpProviderSpec.Node> nodes) {
- return nodes == null ?
- Collections.emptyList() :
- Collections.unmodifiableList(new ArrayList<>(nodes));
- }
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java
deleted file mode 100644
index f86d76a12a4..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/templates/test/TilingTestCase.java
+++ /dev/null
@@ -1,309 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.templates.test;
-
-import com.google.common.util.concurrent.MoreExecutors;
-import com.yahoo.component.chain.Chain;
-import com.yahoo.io.IOUtils;
-import com.yahoo.language.Linguistics;
-import com.yahoo.language.simple.SimpleLinguistics;
-import com.yahoo.prelude.hitfield.XMLString;
-import com.yahoo.prelude.templates.SearchRendererAdaptor;
-import com.yahoo.prelude.templates.TiledTemplateSet;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.Searcher;
-import com.yahoo.search.federation.http.HTTPProviderSearcher;
-import com.yahoo.search.rendering.RendererRegistry;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.result.HitGroup;
-import com.yahoo.search.searchchain.Execution;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests representing a federated and grouped result as a Result object and
- * rendering a tiled output of the result
- *
- * @author bratseth
- */
-@SuppressWarnings("deprecation")
-public class TilingTestCase {
-
- /**
- * This result contains two blocks (center and right).
- * The center block contains multiple subblocks while the right one contains a single block of ads.
- */
- @Test
- public void testTiling() throws IOException {
- Chain<Searcher> chain=new Chain<>("tiling", new TiledResultProducer());
-
- // Query it
- Query query = new Query("/tiled?query=foo");
- Result result = callSearchAndSetRenderer(chain, query);
- assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample.xml")),result);
- }
-
- /**
- * This result contains center section and meta blocks.
- */
- @Test
- public void testTiling2() throws IOException {
- Chain<Searcher> chain= new Chain<>("tiling", new TiledResultProducer2());
-
- // Query it
- Query query=new Query("/tiled?query=foo");
- Result result= callSearchAndSetRenderer(chain, query);
- assertRendered(IOUtils.readFile(new File("src/test/java/com/yahoo/prelude/templates/test/tilingexample2.xml")),result);
- }
-
- private Result callSearchAndSetRenderer(Chain<Searcher> chain, Query query) {
- Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics());
- Result result = new Execution(chain, context).search(query);
- result.getTemplating().setRenderer(new SearchRendererAdaptor(new TiledTemplateSet()));
- return result;
- }
-
- public static void assertRenderedStartsWith(String expected,Result result) throws IOException {
- assertRendered(expected,result,false);
- }
-
- public static void assertRendered(String expected,Result result) throws IOException {
- assertRendered(expected,result,true);
- }
-
- public static void assertRendered(String expected, Result result,boolean checkFullEquality) throws IOException {
- if (checkFullEquality)
- assertEquals(filterComments(expected), getRendered(result));
- else
- assertTrue(getRendered(result).startsWith(expected));
- }
-
- private static String filterComments(String s) {
- StringBuilder b = new StringBuilder();
- for (String line : s.split("\\\n"))
- if ( ! line.startsWith("<!--"))
- b.append(line).append("\n");
- return b.toString();
- }
-
- public static String getRendered(Result result) throws IOException {
- if (result.getTemplating().getRenderer() == null)
- result.getTemplating().setTemplates(null);
-
- // Done in a roundabout way to simulate production code path
- ByteArrayOutputStream stream = new ByteArrayOutputStream();
- Charset cs = Charset.forName(result.getTemplating().getRenderer().getEncoding());
- CharsetDecoder decoder = cs.newDecoder();
- SearchRendererAdaptor.callRender(stream, result);
- stream.flush();
- return decoder.decode(ByteBuffer.wrap(stream.toByteArray())).toString();
- }
-
- private static class TiledResultProducer extends Searcher {
-
- @Override
- public Result search(Query query, Execution execution) {
- Result result = new Result(query);
- result.setTotalHitCount(2800000000l);
-
- // Blocks
- HitGroup centerBlock=(HitGroup)result.hits().add(new HitGroup("section:center"));
- centerBlock.types().add("section");
- centerBlock.setField("region","center");
-
- HitGroup rightBlock=(HitGroup)result.hits().add(new HitGroup("section:right"));
- rightBlock.types().add("section");
- rightBlock.setField("region","right");
-
- // Center groups
- HitGroup navigation=(HitGroup)centerBlock.add(new HitGroup("group:navigation",1.0));
- navigation.types().add("group");
- navigation.setField("type","navigation");
-
- HitGroup adsNorth=(HitGroup)centerBlock.add(new HitGroup("group:ads:north",0.9));
- adsNorth.types().add("group");
- adsNorth.setField("type","ads");
-
- HitGroup hits=(HitGroup)centerBlock.add(new HitGroup("group:navigation",0.8));
- hits.types().add("group");
- hits.setField("type","hits");
-
- HitGroup adsSouth=(HitGroup)centerBlock.add(new HitGroup("group:ads:south",0.7));
- adsSouth.types().add("group");
- adsSouth.setField("type","ads");
-
- // Right group
- HitGroup adsRight=(HitGroup)rightBlock.add(new HitGroup("group:ads:right",0.7));
- adsRight.types().add("group");
- adsRight.setField("type","ads");
-
- // Navigation content
- /*
- Hit alsoTry=navigation.add(new Hit("alsotry"));
- alsoTry.types().add("alsotry");
- alsoTry.setMeta(true);
- LinkList links=new LinkList();
- links.add(new Link("Hilton","?search=Hilton hotel"));
- links.add(new Link("Habbo hotel","?search=Habbo hotel"));
- links.add(new Link("Marriott","?search=Marriott hotel"));
- alsoTry.setField("links",links);
- */
-
- // North ads content
- Hit ad1=adsNorth.add(new Hit("http://www.hotels.com",0.7));
- ad1.types().add("ad");
- ad1.setAuxiliary(true);
- ad1.setField("title",new XMLString("Cheap <hi>hotels</hi>"));
- ad1.setField("body",new XMLString("Low Rates Guaranteed. Call a <hi>Hotel</hi> Expert."));
-
- Hit ad2=adsNorth.add(new Hit("http://www.expedia.com",0.6));
- ad2.types().add("ad");
- ad2.setAuxiliary(true);
- ad2.setField("title",new XMLString("Cheap <hi>hotels</hi> at Expedia"));
- ad2.setField("body","Expedia Special Rates Means We Guarantee Our Low Rates on Rooms.");
-
-// // Hits content
-// // - news hit
-// HitGroup news1=(HitGroup)hits.add(new HitGroup("newsarticles",0.9));
-// news1.setMeta(false);
-// news1.types().add("news");
-// news1.setField("title","Hotel - News results");
-// Hit article1=news1.add(new Hit("www.miamiherald.com/?article=jhsgd7323",0.5));
-// article1.setAuxiliary(true);
-// article1.setField("title","Celebrity blackout: The Hilton of Paris changes name to regain search traffic");
-// article1.types().add("newsarticle");
-// article1.setField("age",23);
-// article1.setField("source","Miami Herald");
-// Hit article2=news1.add(new Hit("www.sfgate.com/?article=8763khj7",0.4));
-// article2.setAuxiliary(true);
-// article2.setField("title","Hotels - more expensive than staying at home");
-// article2.types().add("newsarticle");
-// article2.setField("age",3500);
-// article2.setField("source","SF Gate");
-
- // - collapsed hit
- Hit hit1=hits.add(new Hit("www.hotels.com",0.8));
- hit1.types().add("hit.collapsed");
- hit1.setField("title","Hotels.com | Cheap Hotels | Discount Hotel Rooms | Motels | Lodging");
- hit1.setField("body",new XMLString("Hotels.com helps you find great rates on hotels and discount <hi>hotel</hi> packages."));
- /*
- LinkList collapsed1=new LinkList();
- collapsed1.add(new Link("Last Minute Deals","www.hotels.com/lastminutedeals"));
- collapsed1.add(new Link("Hotel Savings","www.hotels.com/deals"));
- collapsed1.add(new Link("800-994-6835","www.hotels.com/?PSRC=OT2"));
- hit1.setField("links",collapsed1);
- */
-
- // regular hit with links
- Hit hit2=hits.add(new Hit("www.indigohotels.com",0.7));
- hit2.types().add("hit");
- hit2.setField("title","Hotel Indigo Hotels United States - Official Web Site");
- hit2.setField("body","Make Hotel Indigo online hotel reservations and book your hotel rooms today.");
- /*
- LinkList collapsed2=new LinkList();
- collapsed2.add(new Link("800-333-6835","www.indigohotels.com/order"));
- collapsed2.add(new Link("Reservations","www.indigohotels.com/reservations"));
- hit2.setField("links",collapsed2);
- */
-
- // boring old hit
- Hit hit3=hits.add(new Hit("www.all-hotels.com",0.6));
- hit3.types().add("hit");
- hit3.setField("title","All hotels");
- hit3.setField("body","Online hotel directory and reservations.");
-
- // South ads
- Hit southAd1=adsSouth.add(new Hit("www.daysinn.com",1.0));
- southAd1.types().add("ad");
- southAd1.setAuxiliary(true);
- southAd1.setField("title","Days Inn Special Deal");
- southAd1.setField("body","Buy now and Save 15% Off Our Best Available Rate with Days Inn.");
- Hit southAd2=adsSouth.add(new Hit("http://www.expedia.com",0.9));
- southAd2.types().add("ad");
- southAd2.setAuxiliary(true);
- southAd2.setField("title",new XMLString("Cheap <hi>hotels</hi> at Expedia"));
- southAd2.setField("body","Expedia Special Rates Means We Guarantee Our Low Rates on Rooms.");
-
- // Right ads
- Hit rightAd1=adsRight.add(new Hit("www.daysinn.com",1.0));
- rightAd1.types().add("ad");
- rightAd1.setAuxiliary(true);
- rightAd1.setField("title","Days Inn Special Deal");
- rightAd1.setField("body","Buy now and Save 15% Off Our Best Available Rate with Days Inn.");
- Hit rightAd2=adsRight.add(new Hit("www.holidayinn.com",0.9));
- rightAd2.types().add("ad");
- rightAd2.setAuxiliary(true);
- rightAd2.setField("title","Holiday Inn: Official Site");
- rightAd2.setField("body","Book with Holiday Inn. Free Internet. Kids eat free.");
-
- // Done creating result - must analyze because we add ads then later set them as auxiliary
- result.analyzeHits();
-
- return result;
- }
-
- }
-
- private static class TiledResultProducer2 extends Searcher {
-
- @Override
- public Result search(Query query,Execution execution) {
- Result result=new Result(query);
- result.setTotalHitCount(1);
-
- HitGroup section = new HitGroup("section:center");
- result.hits().add(section);
- section.setMeta(false);
- section.types().add("section");
- section.setField("region", "center");
-
- HitGroup yst = new HitGroup("yst");
- section.add(yst);
- yst.setMeta(false);
- yst.setSource("sr");
- yst.types().add("sr");
- yst.setField("provider", "yst");
-
- Hit theHit = new Hit("159");
- yst.add(theHit);
- theHit.setSource("sr");
- theHit.types().add("sr");
- theHit.setField("provider", "yst");
- theHit.setField("title", "Yahoo");
-
- HitGroup meta = new HitGroup("meta");
- result.hits().add(meta);
- meta.types().add("meta");
-
- Hit log = new Hit("com.yahoo.search.federation.yst.YSTBackendSearcherproxy-tw1cache.idp.inktomisearch.com55556/search");
- meta.add(log);
- log.setMeta(true);
- log.setSource("sr");
- log.setField("provider", "yst");
- log.types().add("logging");
- log.setField(HTTPProviderSearcher.LOG_URI, "http://proxy-tw1cache.idp.inktomisearch.com:55556/search?qp=yahootw-twp&Fields=url%2Credirecturl%2Cdate%2Csize%2Cformat%2Csms_product%2Ccacheurl%2Cnodename%2Cid%2Clanguage%2Crsslinks%2Crssvalidatedlinks%2Ccpc%2Cclustertype%2Cxml.active_abstract%2Cactive_abstract_type%2Cactive_abstract_source%2Ccontract_id%2Ctranslated%2Cxml.ydir_tw_hotlist_data%2Cxml.summary%2Cclustercollision%2Cxml.pi_info%2Cpage_adult_overridable%2Cpage_spam_overridable%2Ccategory_ydir%2Chate_edb&Unique=doc%2Chost+2&QueryEncoding=utf-8&Query=ALLWORDS%28yahoo%29&Database=dewownrm-zh-tw&FirstResult=0&srcpvid=&cacheecho=1&ResultsEncoding=utf-8&QueryLanguage=Chinese-traditional&Region=US&NumResults=10&Client=yahoous2");
- log.setField(HTTPProviderSearcher.LOG_SCHEME, "http");
- log.setField(HTTPProviderSearcher.LOG_HOST, "proxy-tw1cache.idp.inktomisearch.com");
- log.setField(HTTPProviderSearcher.LOG_PORT, "55556");
- log.setField(HTTPProviderSearcher.LOG_PATH, "/search");
- log.setField(HTTPProviderSearcher.LOG_STATUS, "200");
- log.setField(HTTPProviderSearcher.LOG_LATENCY_CONNECT, "757");
- log.setField(HTTPProviderSearcher.LOG_RESPONSE_HEADER_PREFIX + "content-length", "16217");
-
- result.analyzeHits();
-
- return result;
- }
-
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java
deleted file mode 100644
index 3e57c48e7f3..00000000000
--- a/container-search/src/test/java/com/yahoo/prelude/test/RankFeatureDumpTestCase.java
+++ /dev/null
@@ -1,74 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.prelude.test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.common.util.concurrent.MoreExecutors;
-import com.yahoo.component.chain.Chain;
-import com.yahoo.language.Linguistics;
-import com.yahoo.language.simple.SimpleLinguistics;
-import com.yahoo.prelude.templates.test.TilingTestCase;
-import com.yahoo.search.rendering.RendererRegistry;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.prelude.fastsearch.FastHit;
-import com.yahoo.search.Searcher;
-import com.yahoo.search.searchchain.Execution;
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests that rank features are rendered when requested in the query
- *
- * @author bratseth
- */
-@SuppressWarnings("deprecation")
-public class RankFeatureDumpTestCase {
-
- private static final String rankFeatureString=
- "{\"match.weight.as1\":10,\"attribute(ai1)\":1.000000,\"proximity(as1, 1, 2)\":2.000000}";
-
- @Test
- public void test() throws IOException {
- Query query=new Query("?query=five&rankfeatures");
- assertTrue(query.getRanking().getListFeatures()); // New api
- Result result = doSearch(new MockBackend(), query, 0,10);
- assertTrue(TilingTestCase.getRendered(result).contains(
- "<field name=\"" + com.yahoo.search.result.Hit.RANKFEATURES_FIELD + "\">" + rankFeatureString + "</field>"));
- }
-
- private static class MockBackend extends Searcher {
-
- @Override
- public Result search(com.yahoo.search.Query query, Execution execution) {
- Result result=new Result(query);
- Hit hit=new FastHit("test",1000);
- hit.setField(com.yahoo.search.result.Hit.RANKFEATURES_FIELD,rankFeatureString);
- result.hits().add(hit);
- return result;
- }
-
- }
-
- private Result doSearch(Searcher searcher, Query query, int offset, int hits) {
- query.setOffset(offset);
- query.setHits(hits);
- return createExecution(searcher).search(query);
- }
-
- private Execution createExecution(Searcher searcher) {
- Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics());
- return new Execution(chainedAsSearchChain(searcher), context);
- }
-
- private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) {
- List<Searcher> searchers = new ArrayList<>();
- searchers.add(topOfChain);
- return new Chain<>(searchers);
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java
deleted file mode 100644
index 6babba5a36a..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/http/GzipDecompressingEntityTestCase.java
+++ /dev/null
@@ -1,212 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import static org.junit.Assert.*;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.UnknownHostException;
-import java.util.Arrays;
-import java.util.Random;
-import java.util.zip.GZIPOutputStream;
-
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.http.message.BasicHeader;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import com.yahoo.text.Utf8;
-
-/**
- * Test GZip support for the HTTP integration introduced in 4.2.
- *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
- */
-public class GzipDecompressingEntityTestCase {
- private static final String STREAM_CONTENT = "00000000000000000000000000000000000000000000000000";
- private static final byte[] CONTENT_AS_BYTES = Utf8.toBytes(STREAM_CONTENT);
- GzipDecompressingEntity testEntity;
-
- private static final class MockEntity implements HttpEntity {
-
- private final InputStream inStream;
-
- MockEntity(InputStream is) {
- inStream = is;
- }
-
- @Override
- public boolean isRepeatable() {
- return false;
- }
-
- @Override
- public boolean isChunked() {
- return false;
- }
-
- @Override
- public long getContentLength() {
- return -1;
- }
-
- @Override
- public Header getContentType() {
- return new BasicHeader("Content-Type", "text/plain");
- }
-
- @Override
- public Header getContentEncoding() {
- return new BasicHeader("Content-Encoding", "gzip");
- }
-
- @Override
- public InputStream getContent() throws IOException,
- IllegalStateException {
- return inStream;
- }
-
- @Override
- public void writeTo(OutputStream outstream) throws IOException {
- }
-
- @Override
- public boolean isStreaming() {
- return false;
- }
-
- @Override
- public void consumeContent() throws IOException {
- }
- }
-
- @Before
- public void setUp() throws Exception {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- GZIPOutputStream gzip = new GZIPOutputStream(out);
- gzip.write(CONTENT_AS_BYTES);
- gzip.finish();
- gzip.close();
- byte[] compressed = out.toByteArray();
- InputStream inStream = new ByteArrayInputStream(compressed);
- testEntity = new GzipDecompressingEntity(new MockEntity(inStream));
- }
-
- @After
- public void tearDown() throws Exception {
- }
-
- @Test
- public final void testGetContentLength() throws UnknownHostException {
- assertEquals(STREAM_CONTENT.length(), testEntity.getContentLength());
- }
-
- @Test
- public final void testGetContent() throws IllegalStateException, IOException {
- InputStream in = testEntity.getContent();
- byte[] buffer = new byte[CONTENT_AS_BYTES.length];
- int read = in.read(buffer);
- assertEquals(CONTENT_AS_BYTES.length, read);
- assertArrayEquals(CONTENT_AS_BYTES, buffer);
- }
-
- @Test
- public final void testGetContentToBigArray() throws IllegalStateException, IOException {
- InputStream in = testEntity.getContent();
- byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2];
- in.read(buffer);
- byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2);
- assertArrayEquals(expected, buffer);
- }
-
- @Test
- public final void testGetContentAvailable() throws IllegalStateException, IOException {
- InputStream in = testEntity.getContent();
- assertEquals(CONTENT_AS_BYTES.length, in.available());
- }
-
- @Test
- public final void testLargeZip() throws IOException {
- byte [] input = new byte [10000000];
- Random random = new Random(89);
- for (int i = 0; i < input.length; i++) {
- input[i] = (byte) random.nextInt();
- }
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- GZIPOutputStream gzip = new GZIPOutputStream(out);
- gzip.write(input);
- gzip.finish();
- gzip.close();
- byte[] compressed = out.toByteArray();
- assertEquals(10003073, compressed.length);
- InputStream inStream = new ByteArrayInputStream(compressed);
- GzipDecompressingEntity gunzipper = new GzipDecompressingEntity(new MockEntity(inStream));
- assertEquals(input.length, gunzipper.getContentLength());
- byte[] buffer = new byte[input.length];
- InputStream content = gunzipper.getContent();
- assertEquals(input.length, content.available());
- int read = content.read(buffer);
- assertEquals(input.length, read);
- assertArrayEquals(input, buffer);
- }
-
- @Test
- public final void testGetContentReadByte() throws IllegalStateException, IOException {
- InputStream in = testEntity.getContent();
- byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2];
- int i = 0;
- while (i < buffer.length) {
- int r = in.read();
- if (r == -1) {
- break;
- } else {
- buffer[i++] = (byte) r;
- }
- }
- byte[] expected = Arrays.copyOf(CONTENT_AS_BYTES, CONTENT_AS_BYTES.length * 2);
- assertEquals(CONTENT_AS_BYTES.length, i);
- assertArrayEquals(expected, buffer);
- }
-
- @Test
- public final void testGetContentReadWithOffset() throws IllegalStateException, IOException {
- InputStream in = testEntity.getContent();
- byte[] buffer = new byte[CONTENT_AS_BYTES.length * 2];
- int read = in.read(buffer, CONTENT_AS_BYTES.length, CONTENT_AS_BYTES.length);
- assertEquals(CONTENT_AS_BYTES.length, read);
- byte[] expected = new byte[CONTENT_AS_BYTES.length * 2];
- for (int i = 0; i < CONTENT_AS_BYTES.length; ++i) {
- expected[CONTENT_AS_BYTES.length + i] = CONTENT_AS_BYTES[i];
- }
- assertArrayEquals(expected, buffer);
- read = in.read(buffer, 0, CONTENT_AS_BYTES.length);
- assertEquals(-1, read);
- }
-
- @Test
- public final void testGetContentSkip() throws IllegalStateException, IOException {
- InputStream in = testEntity.getContent();
- final long n = 5L;
- long skipped = in.skip(n);
- assertEquals(n, skipped);
- int read = in.read();
- assertEquals(CONTENT_AS_BYTES[(int) n], read);
- skipped = in.skip(5000);
- assertEquals(CONTENT_AS_BYTES.length - n - 1, skipped);
- assertEquals(-1L, in.skip(1L));
- }
-
-
- @Test
- public final void testWriteToOutputStream() throws IOException {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- testEntity.writeTo(out);
- assertArrayEquals(CONTENT_AS_BYTES, out.toByteArray());
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java
deleted file mode 100644
index 11f5d7d14fd..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpParametersTest.java
+++ /dev/null
@@ -1,238 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.yahoo.search.federation.ProviderConfig;
-import org.junit.Test;
-
-import static com.yahoo.search.federation.ProviderConfig.Yca;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author gjoranv
- * @author Steinar Knutsen
- */
-public class HttpParametersTest {
-
- @Test
- public void create_from_config() throws Exception {
- ProviderConfig config = new ProviderConfig(new ProviderConfig.Builder()
- .connectionTimeout(1.0)
- .maxConnectionPerRoute(2)
- .maxConnections(3)
- .path("myPath")
- .readTimeout(4)
- .socketBufferBytes(5)
- .yca(new Yca.Builder()
- .applicationId("myId")
- .host("myYcaHost")
- .port(7)
- .retry(8)
- .ttl(9)
- .useProxy(true)));
-
- HTTPParameters httpParameters = new HTTPParameters(config);
-
- // Written to configuredConnectionTimeout, but it is not accessible!?
- //assertThat(httpParameters.getConnectionTimeout(), is(1000));
-
-
- // This value is not set from config by the constructor!?
- //assertThat(httpParameters.getMaxConnectionsPerRoute(), is(2));
-
- // This value is not set from config by the constructor!?
- //assertThat(httpParameters.getMaxTotalConnections(), is(3));
-
- assertThat(httpParameters.getPath(), is("/myPath"));
-
- // This value is not set from config by the constructor!?
- //assertThat(httpParameters.getReadTimeout(), is(4));
-
- // This value is not set from config by the constructor!?
- //assertThat(httpParameters.getSocketBufferSizeBytes(), is(5));
-
-
- assertThat(httpParameters.getYcaUseProxy(), is(true));
- assertThat(httpParameters.getYcaApplicationId(), is("myId"));
- assertThat(httpParameters.getYcaProxy(), is("myYcaHost"));
- assertThat(httpParameters.getYcaPort(), is(7));
- assertThat(httpParameters.getYcaRetry(), is(8000L));
- assertThat(httpParameters.getYcaTtl(), is(9000L));
- }
-
- @Test
- public void requireFreezeWorksForAccessors() {
- HTTPParameters p = new HTTPParameters();
- boolean caught = false;
- final int expected = 37;
- p.setConnectionTimeout(expected);
- assertEquals(expected, p.getConnectionTimeout());
- p.freeze();
- try {
- p.setConnectionTimeout(0);
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setReadTimeout(expected);
- assertEquals(expected, p.getReadTimeout());
- p.freeze();
- try {
- p.setReadTimeout(0);
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setPersistentConnections(true);
- assertTrue(p.getPersistentConnections());
- p.freeze();
- try {
- p.setPersistentConnections(false);
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- assertEquals("http", p.getProxyType());
-
- p = new HTTPParameters();
- caught = false;
- p.setEnableProxy(true);
- assertTrue(p.getEnableProxy());
- p.freeze();
- try {
- p.setEnableProxy(false);
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setProxyHost("nalle");
- assertEquals("nalle", p.getProxyHost());
- p.freeze();
- try {
- p.setProxyHost("jappe");
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setProxyPort(expected);
- assertEquals(expected, p.getProxyPort());
- p.freeze();
- try {
- p.setProxyPort(0);
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setMethod("POST");
- assertEquals("POST", p.getMethod());
- p.freeze();
- try {
- p.setMethod("GET");
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setSchema("gopher");
- assertEquals("gopher", p.getSchema());
- p.freeze();
- try {
- p.setSchema("http");
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setInputEncoding("iso-8859-15");
- assertEquals("iso-8859-15", p.getInputEncoding());
- p.freeze();
- try {
- p.setInputEncoding("shift-jis");
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setOutputEncoding("iso-8859-15");
- assertEquals("iso-8859-15", p.getOutputEncoding());
- p.freeze();
- try {
- p.setOutputEncoding("shift-jis");
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setMaxTotalConnections(expected);
- assertEquals(expected, p.getMaxTotalConnections());
- p.freeze();
- try {
- p.setMaxTotalConnections(0);
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setMaxConnectionsPerRoute(expected);
- assertEquals(expected, p.getMaxConnectionsPerRoute());
- p.freeze();
- try {
- p.setMaxConnectionsPerRoute(0);
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setSocketBufferSizeBytes(expected);
- assertEquals(expected, p.getSocketBufferSizeBytes());
- p.freeze();
- try {
- p.setSocketBufferSizeBytes(0);
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
-
- p = new HTTPParameters();
- caught = false;
- p.setRetries(expected);
- assertEquals(expected, p.getRetries());
- p.freeze();
- try {
- p.setRetries(0);
- } catch (IllegalStateException e) {
- caught = true;
- }
- assertTrue(caught);
- }
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java
deleted file mode 100644
index e0b2afe4e1f..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpPostTestCase.java
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.yahoo.component.ComponentId;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.StupidSingleThreadedHttpServer;
-import com.yahoo.search.federation.ProviderConfig.PingOption;
-import com.yahoo.search.federation.http.Connection;
-import com.yahoo.search.federation.http.HTTPProviderSearcher;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.statistics.Statistics;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.entity.StringEntity;
-import org.junit.Test;
-
-import java.io.InputStream;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.hamcrest.core.StringContains.containsString;
-import static org.junit.Assert.assertThat;
-
-/**
- * See bug #3234696.
- *
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- */
-public class HttpPostTestCase {
-
- @Test
- public void testPostingSearcher() throws Exception {
- StupidSingleThreadedHttpServer server = new StupidSingleThreadedHttpServer();
- server.start();
-
- TestPostSearcher searcher = new TestPostSearcher(new ComponentId("foo:1"),
- Arrays.asList(new Connection("localhost", server.getServerPort())),
- "/");
- Query q = new Query("");
- q.setTimeout(10000000L);
- Execution e = new Execution(searcher, Execution.Context.createContextStub());
-
- searcher.search(q, e);
-
- assertThat(server.getRequest(), containsString("My POST body"));
- server.stop();
- }
-
- private static class TestPostSearcher extends HTTPProviderSearcher {
- public TestPostSearcher(ComponentId id, List<Connection> connections, String path) {
- super(id, connections, httpParameters(path), Statistics.nullImplementation);
- }
-
- private static HTTPParameters httpParameters(String path) {
- HTTPParameters httpParameters = new HTTPParameters(path);
- httpParameters.setPingOption(PingOption.Enum.DISABLE);
- return httpParameters;
- }
-
- @Override
- protected HttpUriRequest createRequest(String method, URI uri, HttpEntity entity) {
- HttpPost request = new HttpPost(uri);
- request.setEntity(entity);
- return request;
- }
-
- @Override
- protected HttpEntity getRequestEntity(Query query, Hit requestMeta) {
- try {
- return new StringEntity("My POST body");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public Map<String, String> getCacheKey(Query q) {
- return new HashMap<>(0);
- }
-
- @Override
- public void unmarshal(final InputStream stream, long contentLength, final Result result) throws IOException {
- // do nothing with the result
- }
-
- @Override
- protected void fill(Result result, String summaryClass, Execution execution, Connection connection) {
- //Empty
- }
- }
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java
deleted file mode 100644
index b4f3c13b8e2..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/http/HttpTestCase.java
+++ /dev/null
@@ -1,124 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.yahoo.component.ComponentId;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.StupidSingleThreadedHttpServer;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.result.HitGroup;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.statistics.Statistics;
-import com.yahoo.text.Utf8;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import javax.xml.bind.JAXBException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Rudimentary http searcher test.
- *
- * @author bratseth
- */
-public class HttpTestCase {
-
- private StupidSingleThreadedHttpServer httpServer;
- private TestHTTPClientSearcher searcher;
-
- @Test
- public void testSearcher() throws JAXBException {
- Result result = searchUsingLocalhost();
-
- assertEquals("ok", result.getQuery().properties().get("gotResponse"));
- assertEquals(0, result.getQuery().errors().size());
- }
-
- private Result searchUsingLocalhost() {
- searcher = new TestHTTPClientSearcher("test","localhost",getPort());
- Query query = new Query("/?query=test");
-
- query.setWindow(0,10);
- return searcher.search(query, new Execution(searcher, Execution.Context.createContextStub()));
- }
-
- @Test
- public void test_that_ip_address_set_on_meta_hit() {
- Result result = searchUsingLocalhost();
- Hit metaHit = getFirstMetaHit(result.hits());
- String ip = (String) metaHit.getField(HTTPSearcher.LOG_IP_ADDRESS);
-
- assertEquals(ip, "127.0.0.1");
- }
-
- private Hit getFirstMetaHit(HitGroup hits) {
- for (Iterator<Hit> i = hits.unorderedDeepIterator(); i.hasNext();) {
- Hit hit = i.next();
- if (hit.isMeta())
- return hit;
- }
- return null;
- }
-
- @Before
- public void setUp() throws Exception {
- httpServer = new StupidSingleThreadedHttpServer(0, 0) {
- @Override
- protected byte[] getResponse(String request) {
- return Utf8.toBytes("HTTP/1.1 200 OK\r\n" +
- "Content-Type: text/xml; charset=UTF-8\r\n" +
- "Connection: close\r\n" +
- "Content-Length: 5\r\n" +
- "\r\n" +
- "hello");
- }
- };
- httpServer.start();
- }
-
- private int getPort() {
- return httpServer.getServerPort();
- }
-
- @After
- public void tearDown() throws Exception {
- httpServer.stop();
- if (searcher != null) {
- searcher.shutdownConnectionManagers();
- }
- }
-
- private static class TestHTTPClientSearcher extends HTTPClientSearcher {
-
- public TestHTTPClientSearcher(String id, String hostName, int port) {
- super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation);
- }
-
- private static List<Connection> toConnections(String hostName,int port) {
- List<Connection> connections=new ArrayList<>();
- connections.add(new Connection(hostName,port));
- return connections;
- }
-
- @Override
- public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException {
- query.properties().set("gotResponse","ok");
- return query;
- }
-
- @Override
- public Map<String, String> getCacheKey(Query q) {
- return null;
- }
-
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java
deleted file mode 100644
index 8c1ff69666b..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/http/PingTestCase.java
+++ /dev/null
@@ -1,288 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.yahoo.component.ComponentId;
-import com.yahoo.prelude.Ping;
-import com.yahoo.prelude.Pong;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.StupidSingleThreadedHttpServer;
-import com.yahoo.search.result.ErrorMessage;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.statistics.Statistics;
-import com.yahoo.text.Utf8;
-import com.yahoo.yolean.Exceptions;
-import org.apache.http.HttpEntity;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Check for different keep-alive scenarios. What we really want to test
- * is the server does not hang.
- *
- * @author Steinar Knutsen
- */
-public class PingTestCase {
-
- private static final int TIMEOUT_MS = 60000;
-
- @Test
- public void testNiceCase() throws Exception {
- NiceStupidServer server = new NiceStupidServer();
- server.start();
- checkSearchAndPing(true, true, true, server.getServerPort());
- server.stop();
- }
-
- private void checkSearchAndPing(boolean firstSearch, boolean pongCheck, boolean secondSearch, int port) {
- String resultThing;
- String comment;
- TestHTTPClientSearcher searcher = new TestHTTPClientSearcher("test",
- "localhost", port);
- try {
-
- Query query = new Query("/?query=test");
-
- query.setWindow(0, 10);
- // high timeout to allow for overloaded test machine
- query.setTimeout(TIMEOUT_MS);
- Ping ping = new Ping(TIMEOUT_MS);
-
- long start = System.currentTimeMillis();
- Execution exe = new Execution(searcher, Execution.Context.createContextStub());
- exe.search(query);
-
- resultThing = firstSearch ? "ok" : null;
- comment = firstSearch ? "First search should have succeeded." : "First search should fail.";
- assertEquals(comment, resultThing, query.properties().get("gotResponse"));
- Pong pong = searcher.ping(ping, searcher.getConnection());
- if (pongCheck) {
- assertEquals("Ping should not have failed.", 0, pong.getErrorSize());
- } else {
- assertEquals("Ping should have failed.", 1, pong.getErrorSize());
- }
- exe = new Execution(searcher, Execution.Context.createContextStub());
- exe.search(query);
-
- resultThing = secondSearch ? "ok" : null;
- comment = secondSearch ? "Second search should have succeeded." : "Second search should fail.";
-
- assertEquals(resultThing, query.properties().get("gotResponse"));
- long duration = System.currentTimeMillis() - start;
- // target for duration based on the timeout values + some slack
- assertTrue("This test probably hanged.", duration < TIMEOUT_MS + 4000);
- searcher.shutdownConnectionManagers();
- } finally {
- searcher.deconstruct();
- }
- }
-
- @Test
- public void testUselessCase() throws Exception {
- UselessStupidServer server = new UselessStupidServer();
- server.start();
- checkSearchAndPing(false, true, false, server.getServerPort());
- server.stop();
- }
-
- @Test
- public void testGrumpyCase() throws Exception {
- GrumpyStupidServer server = new GrumpyStupidServer();
- server.start();
- checkSearchAndPing(false, false, false, server.getServerPort());
- server.stop();
- }
-
- @Test
- public void testPassiveAggressiveCase() throws Exception {
- PassiveAggressiveStupidServer server = new PassiveAggressiveStupidServer();
- server.start();
- checkSearchAndPing(true, false, true, server.getServerPort());
- server.stop();
- }
-
- // OK on ping and search
- private static class NiceStupidServer extends StupidSingleThreadedHttpServer {
- private NiceStupidServer() throws IOException {
- super(0, 0);
- }
-
- @Override
- protected byte[] getResponse(String request) {
- return Utf8.toBytes("HTTP/1.1 200 OK\r\n" +
- "Content-Type: text/xml; charset=UTF-8\r\n" +
- "Connection: close\r\n" +
- "Content-Length: 6\r\n" +
- "\r\n" +
- "hello\n");
- }
- }
-
- // rejects ping and accepts search
- private static class PassiveAggressiveStupidServer extends StupidSingleThreadedHttpServer {
-
- private PassiveAggressiveStupidServer() throws IOException {
- super(0, 0);
- }
-
- @Override
- protected byte[] getResponse(String request) {
- if (request.contains("/ping")) {
- return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" +
- "Content-Type: text/xml; charset=UTF-8\r\n" +
- "Connection: close\r\n" +
- "Content-Length: 8\r\n" +
- "\r\n" +
- "go away\n");
- } else {
- return Utf8.toBytes("HTTP/1.1 200 OK\r\n" +
- "Content-Type: text/xml; charset=UTF-8\r\n" +
- "Connection: close\r\n" +
- "Content-Length: 6\r\n" +
- "\r\n" +
- "hello\n");
- }
- }
- }
-
- // accepts ping and rejects search
- private static class UselessStupidServer extends StupidSingleThreadedHttpServer {
- private UselessStupidServer() throws IOException {
- super(0, 0);
- }
-
-
- @Override
- protected byte[] getResponse(String request) {
- if (request.contains("/ping")) {
- return Utf8.toBytes("HTTP/1.1 200 OK\r\n" +
- "Content-Type: text/xml; charset=UTF-8\r\n" +
- "Connection: close\r\n" +
- "Content-Length: 6\r\n" +
- "\r\n" +
- "hello\n");
- } else {
- return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" +
- "Content-Type: text/xml; charset=UTF-8\r\n" +
- "Connection: close\r\n" +
- "Content-Length: 8\r\n" +
- "\r\n" +
- "go away\n");
- }
- }
- }
-
- // rejects ping and search
- private static class GrumpyStupidServer extends StupidSingleThreadedHttpServer {
- private GrumpyStupidServer() throws IOException {
- super(0, 0);
- }
-
- @Override
- protected byte[] getResponse(String request) {
- return Utf8.toBytes("HTTP/1.1 404 Not found\r\n" +
- "Content-Type: text/xml; charset=UTF-8\r\n" +
- "Connection: close\r\n" +
- "Content-Length: 8\r\n" +
- "\r\n" +
- "go away\n");
- }
- }
-
- private static class TestHTTPClientSearcher extends HTTPClientSearcher {
-
- public TestHTTPClientSearcher(String id, String hostName, int port) {
- super(new ComponentId(id), toConnections(hostName,port), "", Statistics.nullImplementation);
- }
-
- private static List<Connection> toConnections(String hostName,int port) {
- List<Connection> connections=new ArrayList<>();
- connections.add(new Connection(hostName,port));
- return connections;
- }
-
- @Override
- public Query handleResponse(InputStream inputStream, long contentLength, Query query) throws IOException {
- query.properties().set("gotResponse","ok");
- return query;
- }
-
- @Override
- public Result search(Query query, Execution execution,
- Connection connection) {
- URI uri;
- try {
- uri = new URL("http", connection.getHost(), connection
- .getPort(), "/search").toURI();
- } catch (MalformedURLException e) {
- query.errors().add(createMalformedUrlError(query, e));
- return execution.search(query);
- } catch (URISyntaxException e) {
- query.errors().add(createMalformedUrlError(query, e));
- return execution.search(query);
- }
-
- HttpEntity entity;
- try {
- entity = getEntity(uri, query);
- } catch (IOException e) {
- query.errors().add(
- ErrorMessage.createBackendCommunicationError("Error when trying to connect to HTTP backend in "
- + this + " using " + connection
- + " for " + query + ": "
- + Exceptions.toMessageString(e)));
- return execution.search(query);
- } catch (TimeoutException e) {
- query.errors().add(ErrorMessage.createTimeout("No time left for HTTP traffic in "
- + this
- + " for " + query + ": " + e.getMessage()));
- return execution.search(query);
- }
- if (entity == null) {
- query.errors().add(
- ErrorMessage.createBackendCommunicationError("No result from connecting to HTTP backend in "
- + this + " using " + connection + " for " + query));
- return execution.search(query);
- }
-
- try {
- query = handleResponse(entity, query);
- } catch (IOException e) {
- query.errors().add(
- ErrorMessage.createBackendCommunicationError("Error when trying to consume input in "
- + this + ": " + Exceptions.toMessageString(e)));
- } finally {
- cleanupHttpEntity(entity);
- }
- return execution.search(query);
- }
-
- @Override
- public Map<String, String> getCacheKey(Query q) {
- return null;
- }
-
- @Override
- protected URI getPingURI(Connection connection)
- throws MalformedURLException, URISyntaxException {
- return new URL("http", connection.getHost(), connection.getPort(), "/ping").toURI();
- }
-
- Connection getConnection() {
- return getHasher().getNodes().select(0, 0);
- }
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java
deleted file mode 100644
index 19750cf84cc..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/http/QueryParametersTestCase.java
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.http;
-
-import com.yahoo.component.ComponentId;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.statistics.Statistics;
-import com.yahoo.vespa.defaults.Defaults;
-import org.junit.Test;
-
-import java.util.Collections;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Tests that source and backend specific parameters from the query are added correctly to the backend requests
- *
- * @author bratseth
- */
-public class QueryParametersTestCase {
-
- @Test
- public void testQueryParameters() {
- Query query=new Query();
- query.properties().set("a","a-value");
- query.properties().set("b.c","b.c-value");
- query.properties().set("source.otherSource.d","d-value");
- query.properties().set("source.testSource.e","e-value");
- query.properties().set("source.testSource.f.g","f.g-value");
- query.properties().set("provider.testProvider.h","h-value");
- query.properties().set("provider.testProvider.i.j","i.j-value");
-
- query.properties().set("sourceName","testSource"); // Done by federation searcher
- query.properties().set("providerName","testProvider"); // Done by federation searcher
-
- TestHttpProvider searcher=new TestHttpProvider();
- Map<String,String> parameters=searcher.getQueryMap(query);
- searcher.deconstruct();
-
- assertEquals(4,parameters.size()); // the appropriate 4 of the above
- assertEquals(parameters.get("e"),"e-value");
- assertEquals(parameters.get("f.g"),"f.g-value");
- assertEquals(parameters.get("h"),"h-value");
- assertEquals(parameters.get("i.j"),"i.j-value");
- }
-
- public static class TestHttpProvider extends HTTPProviderSearcher {
-
- public TestHttpProvider() {
- super(new ComponentId("test"), Collections.singletonList(new Connection("host", Defaults.getDefaults().vespaWebServicePort())), "path", Statistics.nullImplementation);
- }
-
- @Override
- public Map<String, String> getCacheKey(Query q) {
- return Collections.singletonMap("nocaching", String.valueOf(Math.random()));
- }
-
- @Override
- protected void fill(Result result, String summaryClass, Execution execution, Connection connection) {
- }
-
- }
-
-}
-
diff --git a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/image/.gitignore
+++ /dev/null
diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java
deleted file mode 100644
index 022177bc42b..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryMarshallerTestCase.java
+++ /dev/null
@@ -1,160 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.vespa.test;
-
-import com.yahoo.language.Linguistics;
-import com.yahoo.language.simple.SimpleLinguistics;
-import com.yahoo.prelude.IndexFacts;
-import com.yahoo.prelude.query.*;
-import com.yahoo.search.Query;
-import com.yahoo.search.federation.vespa.QueryMarshaller;
-import com.yahoo.search.searchchain.Execution;
-import com.yahoo.search.test.QueryTestCase;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class QueryMarshallerTestCase {
-
- private static final Linguistics linguistics = new SimpleLinguistics();
-
- @Test
- public void testCommonCommonCase() {
- AndItem root = new AndItem();
- addThreeWords(root);
- assertEquals("a AND b AND c", new QueryMarshaller().marshal(root));
- }
-
- @Test
- public void testPhrase() {
- PhraseItem root = new PhraseItem();
- root.setIndexName("habla");
- addThreeWords(root);
- assertEquals("habla:\"a b c\"", new QueryMarshaller().marshal(root));
- }
-
- @Test
- public void testPhraseDefaultIndex() {
- PhraseItem root = new PhraseItem();
- addThreeWords(root);
- assertEquals("\"a b c\"", new QueryMarshaller().marshal(root));
- }
-
- @Test
- public void testLittleMoreComplex() {
- AndItem root = new AndItem();
- addThreeWords(root);
- OrItem ambig = new OrItem();
- root.addItem(ambig);
- addThreeWords(ambig);
- AndItem but = new AndItem();
- addThreeWords(but);
- ambig.addItem(but);
- assertEquals("a AND b AND c AND ( a OR b OR c OR ( a AND b AND c ) )",
- new QueryMarshaller().marshal(root));
- }
-
- @Test
- public void testRank() {
- RankItem root = new RankItem();
- addThreeWords(root);
- assertEquals("a RANK b RANK c", new QueryMarshaller().marshal(root));
- }
-
- @Test
- public void testNear() {
- NearItem near = new NearItem(3);
- addThreeWords(near);
- assertEquals("a NEAR(3) b NEAR(3) c", new QueryMarshaller().marshal(near));
- }
-
- @Test
- public void testONear() {
- ONearItem oNear = new ONearItem(3);
- addThreeWords(oNear);
- assertEquals("a ONEAR(3) b ONEAR(3) c", new QueryMarshaller().marshal(oNear));
- }
-
- private void addThreeWords(CompositeItem root) {
- root.addItem(new WordItem("a"));
- root.addItem(new WordItem("b"));
- root.addItem(new WordItem("c"));
- }
-
- @Test
- public void testNegativeGroupedTerms() {
- testQueryString(new QueryMarshaller(), "a -(b c) -(d e)",
- "a ANDNOT ( b AND c ) ANDNOT ( d AND e )");
- }
-
- @Test
- public void testPositiveGroupedTerms() {
- testQueryString(new QueryMarshaller(), "a (b c)", "a AND ( b OR c )");
- }
-
- @Test
- public void testInt() {
- testQueryString(new QueryMarshaller(), "yahoo 123", "yahoo AND 123");
- }
-
- @Test
- public void testCJKOneWord() {
- testQueryString(new QueryMarshaller(), "天龍人");
- }
-
- @Test
- public void testTwoWords() {
- testQueryString(new QueryMarshaller(), "John Smith", "John AND Smith", null, new SimpleLinguistics());
- }
-
- @Test
- public void testTwoWordsInPhrase() {
- testQueryString(new QueryMarshaller(), "\"John Smith\"", "\"John Smith\"", null, new SimpleLinguistics());
- }
-
- @Test
- public void testCJKTwoSentences() {
- testQueryString(new QueryMarshaller(), "是不是這樣的夜晚 你才會這樣地想起我", "是不是這樣的夜晚 AND 你才會這樣地想起我");
- }
-
- @Test
- public void testCJKTwoSentencesWithLanguage() {
- testQueryString(new QueryMarshaller(), "助妳好孕 生1胎北市發2萬", "助妳好孕 AND 生1胎北市發2萬", "zh-Hant");
- }
-
- @Test
- public void testCJKTwoSentencesInPhrase() {
- QueryMarshaller marshaller = new QueryMarshaller();
- testQueryString(marshaller, "\"助妳好孕 生1胎北市發2萬\"", "\"助妳好孕 生1胎北市發2萬\"", "zh-Hant");
- testQueryString(marshaller, "\"是不是這樣的夜晚 你才會這樣地想起我\"", "\"是不是這樣的夜晚 你才會這樣地想起我\"");
- }
-
- @Test
- public void testCJKMultipleSentences() {
- testQueryString(new QueryMarshaller(), "염부장님과 함께했던 좋은 추억들은", "염부장님과 AND 함께했던 AND 좋은 AND 추억들은");
- }
-
- @Test
- public void testIndexRestriction() {
- /** ticket 3707606, comment #29 */
- testQueryString(new QueryMarshaller(), "site:nytimes.com", "site:\"nytimes com\"");
- }
-
- private void testQueryString(QueryMarshaller marshaller, String uq) {
- testQueryString(marshaller, uq, uq, null);
- }
-
- private void testQueryString(QueryMarshaller marshaller, String uq, String mq) {
- testQueryString(marshaller, uq, mq, null);
- }
-
- private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang) {
- testQueryString(marshaller, uq, mq, lang, linguistics);
- }
-
- private void testQueryString(QueryMarshaller marshaller, String uq, String mq, String lang, Linguistics linguistics) {
- Query query = new Query("/?query=" + QueryTestCase.httpEncode(uq) + ((lang != null) ? "&language=" + lang : ""));
- query.getModel().setExecution(new Execution(new Execution.Context(null, new IndexFacts(), null, null, linguistics)));
- assertEquals(mq, marshaller.marshal(query.getModel().getQueryTree().getRoot()));
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java
deleted file mode 100644
index 8d1ecf4085e..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/QueryParametersTestCase.java
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.vespa.test;
-
-import com.yahoo.search.Query;
-import com.yahoo.search.federation.vespa.VespaSearcher;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Tests that source and backend specific parameters from the query are added correctly to the backend requests
- *
- * @author bratseth
- */
-public class QueryParametersTestCase {
-
- public void testQueryParameters() {
- Query query=new Query();
- query.properties().set("a","a-value");
- query.properties().set("b.c","b.c-value");
- query.properties().set("source.otherSource.d","d-value");
- query.properties().set("source.testSource.e","e-value");
- query.properties().set("source.testSource.f.g","f.g-value");
- query.properties().set("provider.testProvider.h","h-value");
- query.properties().set("provider.testProvider.i.j","i.j-value");
-
- query.properties().set("sourceName","testSource"); // Done by federation searcher
- query.properties().set("providerName","testProvider"); // Done by federation searcher
-
- VespaSearcher searcher=new VespaSearcher("testProvider","",0,"");
- Map<String,String> parameters=searcher.getQueryMap(query);
- searcher.deconstruct();
-
- assertEquals(9, parameters.size()); // 5 standard + the appropriate 4 of the above
- assertEquals(parameters.get("e"),"e-value");
- assertEquals(parameters.get("f.g"),"f.g-value");
- assertEquals(parameters.get("h"),"h-value");
- assertEquals(parameters.get("i.j"),"i.j-value");
- }
-
-}
-
diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java
deleted file mode 100644
index 50186e03ae8..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/ResultBuilderTestCase.java
+++ /dev/null
@@ -1,94 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.vespa.test;
-
-import java.util.Iterator;
-
-import com.yahoo.net.URI;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.federation.vespa.ResultBuilder;
-import com.yahoo.search.result.ErrorHit;
-import com.yahoo.search.result.ErrorMessage;
-import com.yahoo.search.result.HitGroup;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test XML parsing of results.
- *
- * @author Steinar Knutsen
- */
-@SuppressWarnings("deprecation")
-public class ResultBuilderTestCase {
-
- private boolean quickCompare(double a, double b) {
- double z = Math.min(Math.abs(a), Math.abs(b));
- if (Math.abs((a - b)) < (z / 1e14)) {
- return true;
- } else {
- return false;
- }
- }
-
- @Test
- public void testSimpleResult() {
- boolean gotErrorDetails = false;
- ResultBuilder r = new ResultBuilder();
- Result res = r.parse("file:src/test/java/com/yahoo/prelude/searcher/test/testhit.xml", new Query("?query=a"));
- assertEquals(3, res.getConcreteHitCount());
- assertEquals(4, res.getHitCount());
- ErrorHit e = (ErrorHit) res.hits().get(0);
- // known problem, if the same error is the main error is
- // in details, it'll be added twice. Not sure how to fix that,
- // because old Vespa systems give no error details, and there
- // is no way of nuking an existing error if the details exist.
- for (Iterator<?> i = e.errorIterator(); i.hasNext();) {
- ErrorMessage err = (ErrorMessage) i.next();
- assertEquals(5, err.getCode());
- String details = err.getDetailedMessage();
- if (details != null) {
- gotErrorDetails = true;
- assertEquals("An error as ordered", details.trim());
- }
- }
- assertTrue("Error details are missing", gotErrorDetails);
- assertEquals(new URI("http://def"), res.hits().get(1).getId());
- assertEquals("test/stuff\\tsome/other", res.hits().get(2).getField("category"));
- assertEquals("<field>habla</field>"
- + "<hi>blbl</hi><br />&lt;&gt;&amp;fdlkkgj&lt;/field&gt;;lk<a b=\"1\" c=\"2\" />"
- + "<x><y><z /></y></x>", res.hits().get(3).getField("annoying").toString());
- }
-
- @Test
- public void testNestedResult() {
- ResultBuilder r = new ResultBuilder();
- Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml", new Query("?query=a"));
- assertNull(res.hits().getError());
- assertEquals(3, res.hits().size());
- assertEquals("ABCDEFGHIJKLMNOPQRSTUVWXYZ", res.hits().get(0).getField("guid").toString());
- HitGroup g1 = (HitGroup) res.hits().get(1);
- HitGroup g2 = (HitGroup) res.hits().get(2);
- assertEquals(15, g1.size());
- assertEquals("reward_for_thumb", g1.get(1).getField("id").toString());
- assertEquals(10, g2.size());
- HitGroup g3 = (HitGroup) g2.get(3);
- assertEquals("badge", g3.types().iterator().next());
- assertEquals(2, g3.size());
- assertEquals("badge/Topic Explorer 5", g3.get(0).getField("name").toString());
- }
-
- @Test
- public void testWeirdDocumentID() {
- ResultBuilder r = new ResultBuilder();
- Result res = r.parse("file:src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml", new Query("?query=a"));
- assertNull(res.hits().getError());
- assertEquals(3, res.hits().size());
- assertEquals(new URI("nalle"), res.hits().get(0).getId());
- assertEquals(new URI("tralle"), res.hits().get(1).getId());
- assertEquals(new URI("kalle"), res.hits().get(2).getId());
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java
deleted file mode 100644
index 7fbe883bfb7..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaIntegrationTestCase.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.vespa.test;
-
-import com.yahoo.component.chain.Chain;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.Searcher;
-import com.yahoo.search.federation.vespa.VespaSearcher;
-import com.yahoo.search.searchchain.Execution;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author bratseth
- */
-public class VespaIntegrationTestCase {
-
- // TODO: Setup the answering vespa searcher from this test....
- @Test
- public void testIt() {
- if (System.currentTimeMillis() > 0) return;
- Chain<Searcher> chain = new Chain<>(new VespaSearcher("test","example.yahoo.com",19010,""));
- Result result = new Execution(chain, Execution.Context.createContextStub()).search(new Query("?query=test"));
- assertEquals(23, result.hits().size());
- }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java
deleted file mode 100644
index cc00c6739fa..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/VespaSearcherTestCase.java
+++ /dev/null
@@ -1,240 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.federation.vespa.test;
-
-import com.yahoo.prelude.query.*;
-import com.yahoo.search.Query;
-import com.yahoo.search.federation.vespa.VespaSearcher;
-import com.yahoo.search.query.QueryTree;
-import com.yahoo.search.query.parser.Parsable;
-import com.yahoo.search.query.parser.Parser;
-import com.yahoo.search.query.parser.ParserEnvironment;
-import com.yahoo.search.query.parser.ParserFactory;
-import com.yahoo.search.result.Hit;
-import com.yahoo.search.searchchain.Execution;
-import org.apache.http.HttpEntity;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.net.URI;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Check query marshaling in VespaSearcher works.
- *
- * @author Steinar Knutsen
- */
-public class VespaSearcherTestCase {
-
- private VespaSearcher searcher;
-
- @Before
- public void setUp() {
- searcher = new VespaSearcher("cache1","",0,"");
- }
-
- @After
- public void tearDown() {
- searcher.deconstruct();
- }
-
- @Test
- public void testMarshalQuery() {
- RankItem root = new RankItem();
- QueryTree r = new QueryTree(root);
- AndItem recall = new AndItem();
- PhraseItem usual = new PhraseItem();
- PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"});
- WordItem filterWord = new WordItem("silly");
-
- filterPhrase.setFilter(true);
- filterWord.setFilter(true);
-
- root.addItem(recall);
- usual.addItem(new WordItem("new"));
- usual.addItem(new WordItem("york"));
- recall.addItem(usual);
- recall.addItem(new WordItem("shoes"));
- root.addItem(new WordItem("nike"));
- root.addItem(new WordItem("adidas"));
- root.addItem(filterPhrase);
- recall.addItem(filterWord);
-
- assertEquals("( \"new york\" AND shoes AND silly ) RANK nike RANK adidas RANK \"bloody expensive\"", searcher.marshalQuery(r));
- }
-
- @Test
- public void testMarshalQuerySmallTree() {
- RankItem root = new RankItem();
- QueryTree r = new QueryTree(root);
- AndItem recall = new AndItem();
- PhraseItem usual = new PhraseItem();
- PhraseItem filterPhrase = new PhraseItem(new String[] {"bloody", "expensive"});
- WordItem filterWord = new WordItem("silly");
-
- filterPhrase.setFilter(true);
- filterWord.setFilter(true);
-
- root.addItem(recall);
- usual.addItem(new WordItem("new"));
- usual.addItem(new WordItem("york"));
- recall.addItem(usual);
- recall.addItem(new WordItem("shoes"));
- root.addItem(filterPhrase);
- recall.addItem(filterWord);
-
- assertEquals("( \"new york\" AND shoes AND silly ) RANK \"bloody expensive\"", searcher.marshalQuery(r));
- // TODO: Switch to this 2-way check rather than just 1-way and then also make this actually treat filter terms correctly
- // assertMarshals(root)
- }
-
- @Test
- public void testWandMarshalling() {
- WeakAndItem root = new WeakAndItem();
- root.setN(32);
- root.addItem(new WordItem("a"));
- root.addItem(new WordItem("b"));
- root.addItem(new WordItem("c"));
- assertMarshals(root);
- }
-
- @Test
- public void testWandMarshalling2() {
- // AND (WAND(10) a!1 the!10) source:yahoonews
- AndItem root = new AndItem();
- WeakAndItem wand = new WeakAndItem(10);
- wand.addItem(newWeightedWordItem("a",1));
- wand.addItem(newWeightedWordItem("the",10));
- root.addItem(wand);
- root.addItem(new WordItem("yahoonews","source"));
- assertMarshals(root);
- }
-
- private WordItem newWeightedWordItem(String word,int weight) {
- WordItem wordItem=new WordItem(word);
- wordItem.setWeight(weight);
- return wordItem;
- }
-
- private void assertMarshals(Item root) {
- QueryTree r = new QueryTree(root);
- String marshalledQuery=searcher.marshalQuery(r);
- assertEquals("Marshalled form '" + marshalledQuery + "' recreates the original",
- r,parseQuery(marshalledQuery,""));
- }
-
- private static Item parseQuery(String query, String filter) {
- Parser parser = ParserFactory.newInstance(Query.Type.ADVANCED, new ParserEnvironment());
- return parser.parse(new Parsable().setQuery(query).setFilter(filter));
- }
-
- @Test
- public void testSourceProviderProperties() throws Exception {
- /* TODO: update test
- Server httpServer = new Server();
- try {
- SocketConnector listener = new SocketConnector();
- listener.setHost("0.0.0.0");
- httpServer.addConnector(listener);
- httpServer.setHandler(new DummyHandler());
- httpServer.start();
-
- int port=httpServer.getConnectors()[0].getLocalPort();
-
- List<SourcesConfig.Source> sourcesConfig = new ArrayList<SourcesConfig.Source>();
- SourcesConfig.Source sourceConfig = new SourcesConfig.Source();
- sourceConfig.chain.setValue("news");
- sourceConfig.provider.setValue("news");
- sourceConfig.id.setValue("news");
- sourceConfig.timelimit.value = 10000;
- sourcesConfig.add(sourceConfig);
- FederationSearcher federator =
- new FederationSearcher(ComponentId.createAnonymousComponentId(),
- new ArrayList<SourcesConfig.Source>(sourcesConfig));
- SearchChain mainChain=new OrderedSearchChain(federator);
-
- SearchChainRegistry registry=new SearchChainRegistry();
- SearchChain sourceChain=new SearchChain(new ComponentId("news"),new VespaSearcher("test","localhost",port,""));
- registry.register(sourceChain);
- Query query=new Query("?query=hans&hits=20&provider.news.a=a1&source.news.b=b1");
- Result result=new Execution(mainChain,registry).search(query);
- assertNull(result.hits().getError());
- Hit testHit=result.hits().get("testHit");
- assertNotNull(testHit);
- assertEquals("testValue",testHit.fields().get("testField"));
- assertEquals("a1",testHit.fields().get("a"));
- assertEquals("b1",testHit.fields().get("b"));
- }
- finally {
- httpServer.stop();
- }
- */
- }
-
- @Test
- public void testVespaSearcher() {
- VespaSearcher v=new VespaSearcherValidatingSubclass();
- new Execution(v, Execution.Context.createContextStub()).search(new Query(com.yahoo.search.test.QueryTestCase.httpEncode("?query=test&filter=myfilter")));
- }
-
- private class VespaSearcherValidatingSubclass extends VespaSearcher {
-
- public VespaSearcherValidatingSubclass() {
- super("configId","host",80,"path");
- }
-
- @Override
- protected HttpEntity getEntity(URI uri, Hit requestMeta, Query query) throws IOException {
- assertEquals("http://host:80/path?query=test+RANK+myfilter&type=adv&offset=0&hits=10&presentation.format=xml",uri.toString());
- return super.getEntity(uri,requestMeta,query);
- }
-
- }
-
- // used by the old testSourceProviderProperties()
-// private class DummyHandler extends AbstractHandler {
-// public void handle(String s, Request request, HttpServletRequest httpServletRequest,
-// HttpServletResponse httpServletResponse) throws IOException, ServletException {
-//
-// try {
-// Response httpResponse = httpServletResponse instanceof Response ? (Response) httpServletResponse : HttpConnection.getCurrentConnection().getResponse();
-//
-// httpResponse.setStatus(HttpStatus.OK_200);
-// httpResponse.setContentType("text/xml");
-// httpResponse.setCharacterEncoding("UTF-8");
-// Result r=new Result(new Query());
-// Hit testHit=new Hit("testHit");
-// testHit.setField("uri","testHit"); // That this is necessary is quite unfortunate...
-// testHit.setField("testField","testValue");
-// // Write back all incoming properties:
-// for (Object e : httpServletRequest.getParameterMap().entrySet()) {
-// Map.Entry entry=(Map.Entry)e;
-// testHit.setField(entry.getKey().toString(),getFirstValue(entry.getValue()));
-// }
-//
-// r.hits().add(testHit);
-//
-// //StringWriter sw=new StringWriter();
-// //r.render(sw);
-// //System.out.println(sw.toString());
-//
-// SearchRendererAdaptor.callRender(httpResponse.getWriter(), r);
-// httpResponse.complete();
-// }
-// catch (Exception e) {
-// System.out.println("WARNING: Could not respond to request: " + Exceptions.toMessageString(e));
-// e.printStackTrace();
-// }
-// }
-//
-// private String getFirstValue(Object entry) {
-// if (entry instanceof String[])
-// return ((String[])entry)[0].toString();
-// else
-// return entry.toString();
-// }
-// }
-
-}
diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml
deleted file mode 100644
index c17bbf8474d..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/idhits.xml
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="utf-8" ?>
-<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-<result total-hit-count="3">
- <hit relevancy="75" source="test" type="summary">
- <field name="uri">nalle</field>
- <field name="relevancy">75</field>
- <field name="collapseId">0</field>
- </hit>
- <hit relevancy="73" source="test" type="summary test other">
- <field name="documentId">tralle</field>
- <field name="relevancy">73</field>
- <field name="collapseId">0</field>
- <field name="category">test/stuff\tsome/other</field>
- <field name="bsumtitle">dklf øæå sdf &gt; &amp; &lt;
-Ipsum, etc.</field>
- </hit>
- <hit relevancy="70" source="test" type="summary">
- <field name="DOCUMENTID">kalle</field>
- <field name="relevancy">75</field>
- <field name="collapseId">0</field>
- <field name="annoying"><field>habla</field><hi>blbl</hi><br /><![CDATA[<>&fdlkkgj</field>]]>;lk<a b="1" c="2" /><x><y><z /></y></x></field>
- </hit>
-</result>
diff --git a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml b/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml
deleted file mode 100644
index 8b5ab710378..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/vespa/test/nestedhits.xml
+++ /dev/null
@@ -1,318 +0,0 @@
-<?xml version="1.0" encoding="utf-8" ?>
-<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -->
-<result total-hit-count="36">
-<hit type="user_reputation">
-<field name="guid">ABCDEFGHIJKLMNOPQRSTUVWXYZ</field>
-<field name="level">zero</field>
-<field name="points">0</field>
-<field name="created">1287600988</field>
-<field name="updated">1287600988</field>
-</hit>
-<group type="actions">
-<hit type="action">
-<field name="id">thumb</field>
-<field name="created">1287600992</field>
-<field name="updated">1287600992</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-<hit type="action">
-<field name="id">reward_for_thumb</field>
-<field name="created">1287600992</field>
-<field name="updated">1287600992</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-<hit type="action">
-<field name="id">undo_thumb</field>
-<field name="created">1287600992</field>
-<field name="updated">1287600992</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">buzz</field>
-<field name="created">1287600989</field>
-<field name="updated">1287600989</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">undo_reward_for_thumb</field>
-<field name="created">1287600992</field>
-<field name="updated">1287600992</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">vote</field>
-<field name="created">1287600989</field>
-<field name="updated">1287600989</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">report_abuse</field>
-<field name="created">1287600992</field>
-<field name="updated">1287600992</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">reward_for_vote</field>
-<field name="created">1287600989</field>
-<field name="updated">1287600989</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">signup</field>
-<field name="created">1287600993</field>
-<field name="updated">1287600993</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">registered</field>
-<field name="created">1287600989</field>
-<field name="updated">1287600989</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">get_points</field>
-<field name="created">1287600989</field>
-<field name="updated">1287600989</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">contrib_SignedUp</field>
-<field name="created">1287600993</field>
-<field name="updated">1287600993</field>
-<field name="points">0</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">contrib_AgreedToTos</field>
-<field name="created">1287600993</field>
-<field name="updated">1287600993</field>
-<field name="points">500</field>
-<field name="level">zero</field>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">Create Feature</field>
-<field name="created"/>
-<field name="updated"/>
-<field name="points">0</field>
-<field name="level"/>
-<field name="isEnabled">1</field>
-</hit>
-
-<hit type="action">
-<field name="id">add_theme</field>
-<field name="created"/>
-<field name="updated"/>
-<field name="points">0</field>
-<field name="level"/>
-<field name="isEnabled">1</field>
-</hit>
-</group>
-
-<group type="awards">
-
-<group type="badge">
-
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/First Feature</field>
-<field name="description">You’ve created your First Feature!</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/1stfeature.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-
-<group type="badge">
-
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/25th Feature</field>
-<field name="description">You’ve created your 25th Feature!</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/25thfeature.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-
-<group type="badge">
-
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/50th Feature</field>
-<field name="description">You’ve created your 50th Feature!</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/10thfeature.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-
-<group type="badge">
-
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/Topic Explorer 5</field>
-<field name="description">You’ve added a Feature to your 5th Topic Page!</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/5thtopic.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-
-<group type="badge">
-
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/Topic Explorer 15</field>
-<field name="description">You’ve added a Feature to your 15th Topic Page!</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/15thtopic.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-
-<group type="badge">
-
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/Topic Explorer 30</field>
-<field name="description">You’ve added a Feature to your 30th Topic Page!</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/30thtopic.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-
-<group type="badge">
-
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/Pollster</field>
-<field name="description">You’ve created your 5th Poll Feature.</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/pollster.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-<group type="badge">
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/Reporter</field>
-<field name="description">You’ve created your 5th Article Feature.</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/newsreporter.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-<group type="badge">
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/Paparazzi</field>
-<field name="description">You’ve created your 5th Image Feature.</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/paparazzi.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-<group type="badge">
-<hit type="info">
-<field name="type">badge</field>
-<field name="name">badge/Video Reporter</field>
-<field name="description">You’ve created your 5th Video Feature.</field>
-<field name="status">active</field>
-<field name="imageUrl">http://example.yahoo.com/director.png</field>
-<field name="imageHeight">57</field>
-<field name="imageWidth">57</field>
-</hit>
-<hit type="earned">
-<field name="date">1283981088</field>
-<field name="context">topic/Jennifer_Aniston</field>
-</hit>
-</group>
-</group>
-</result>
diff --git a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore b/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore
deleted file mode 100644
index e69de29bb2d..00000000000
--- a/container-search/src/test/java/com/yahoo/search/federation/ysm/.gitignore
+++ /dev/null
diff --git a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java
index 999a6d32ac6..569f219f635 100644
--- a/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/pagetemplates/engine/test/ExecutionAbstractTestCase.java
@@ -3,9 +3,6 @@ package com.yahoo.search.pagetemplates.engine.test;
import com.google.common.util.concurrent.ListenableFuture;
import com.yahoo.io.IOUtils;
-import com.yahoo.prelude.templates.TiledTemplateSet;
-import com.yahoo.prelude.templates.UserTemplate;
-import com.yahoo.prelude.templates.test.TilingTestCase;
import com.yahoo.search.Result;
import com.yahoo.search.pagetemplates.PageTemplate;
import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader;
diff --git a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java b/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java
deleted file mode 100644
index 11d163373ac..00000000000
--- a/container-search/src/test/java/com/yahoo/search/result/test/ArrayOutputTestCase.java
+++ /dev/null
@@ -1,35 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.search.result.test;
-
-import java.io.IOException;
-
-import com.yahoo.prelude.hitfield.XMLString;
-import com.yahoo.prelude.templates.test.TilingTestCase;
-import com.yahoo.search.Query;
-import com.yahoo.search.Result;
-import com.yahoo.search.result.Hit;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * @author bratseth
- */
-public class ArrayOutputTestCase {
-
- @Test
- public void testArrayOutput() throws IOException {
- Result r=new Result(new Query("?query=ignored"));
- Hit hit=new Hit("test");
- hit.setField("phone",new XMLString("\n <item>408-555-1234</item>" + "\n <item>408-555-5678</item>\n "));
- r.hits().add(hit);
-
- String rendered = TilingTestCase.getRendered(r);
- String[] lines= rendered.split("\n");
- assertEquals(" <field name=\"phone\">",lines[4]);
- assertEquals(" <item>408-555-1234</item>",lines[5]);
- assertEquals(" <item>408-555-5678</item>",lines[6]);
- assertEquals(" </field>",lines[7]);
- }
-
-}