diff options
| author | Jon Bratseth <bratseth@gmail.com> | 2022-06-05 13:04:17 +0200 |
|---|---|---|
| committer | gjoranv <gv@verizonmedia.com> | 2022-06-08 11:45:30 +0200 |
| commit | 8647f18a80c40c3fe98f7f68e0330683786eb50d (patch) | |
| tree | 9cb2a4bb9658f40de10b6ded892b54f928ed6e6f | |
| parent | eabb1d3195a1bd6d34def333854ad713bd6ac886 (diff) | |
Remove vespa-http-client on Vespa 8
98 files changed, 0 insertions, 11271 deletions
@@ -135,7 +135,6 @@ <module>vespa-feed-client-api</module> <module>vespa-feed-client-cli</module> <module>vespa-hadoop</module> - <module>vespa-http-client</module> <module>vespa-maven-plugin</module> <module>vespa-osgi-testrunner</module> <module>vespa-testrunner-components</module> diff --git a/vespa-http-client/.gitignore b/vespa-http-client/.gitignore deleted file mode 100644 index 12251442258..00000000000 --- a/vespa-http-client/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target -/pom.xml.build diff --git a/vespa-http-client/CMakeLists.txt b/vespa-http-client/CMakeLists.txt deleted file mode 100644 index 3fb22316a28..00000000000 --- a/vespa-http-client/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_jar(vespa-http-client-jar-with-dependencies.jar) diff --git a/vespa-http-client/OWNERS b/vespa-http-client/OWNERS deleted file mode 100644 index dbcff24b338..00000000000 --- a/vespa-http-client/OWNERS +++ /dev/null @@ -1 +0,0 @@ -vekterli diff --git a/vespa-http-client/README b/vespa-http-client/README deleted file mode 100644 index 49d375316f6..00000000000 --- a/vespa-http-client/README +++ /dev/null @@ -1,5 +0,0 @@ -Java API for feeding to Vespa from external sources - -Cross-colo etc. - -NOTE: This must be built with JDK 8, because it's used on the grid. diff --git a/vespa-http-client/abi-spec.json b/vespa-http-client/abi-spec.json deleted file mode 100644 index 9e26dfeeb6e..00000000000 --- a/vespa-http-client/abi-spec.json +++ /dev/null @@ -1 +0,0 @@ -{}
\ No newline at end of file diff --git a/vespa-http-client/pom.xml b/vespa-http-client/pom.xml deleted file mode 100644 index 0448b2b862b..00000000000 --- a/vespa-http-client/pom.xml +++ /dev/null @@ -1,219 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>parent</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../parent/pom.xml</relativePath> - </parent> - <artifactId>vespa-http-client</artifactId> - <version>7-SNAPSHOT</version> - <name>${project.artifactId}</name> - <description>Independent external feeding API towards Vespa.</description> - - <dependencies> - - <!-- NOTE: Adding dependencies here may break clients because this is used outside an OSGi container with - manually set up classpaths and possibly no access to the with-dependencies jar. - - Vespa dependencies should not be added. - --> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-xml</artifactId> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>jackson-datatype-jsr310</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>com.fasterxml.jackson.datatype</groupId> - <artifactId>jackson-datatype-jdk8</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>io.airlift</groupId> - <artifactId>airline</artifactId> - <version>0.6</version> - </dependency> - <dependency> - <!-- Needed for Vespa TLS configuration. Standard jar artifact --> - <groupId>com.yahoo.vespa</groupId> - <artifactId>http-utils</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - <exclusions> - <!-- Optimization: exclude artifacts that are not used by this module --> - <exclusion> - <groupId>io.airlift</groupId> - <artifactId>aircompressor</artifactId> - </exclusion> - <exclusion> - <groupId>net.java.dev.jna</groupId> - <artifactId>jna</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </exclusion> - <exclusion> - <groupId>org.lz4</groupId> - <artifactId>lz4-java</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <!-- Needed for Vespa TLS configuration. --> - <!-- This artifact is packaged as an OSGi bundle - make sure to manually include or exclude transitive dependencies as necessary --> - <!-- Note: includes BouncyCastle to compile scope transitively. --> - <groupId>com.yahoo.vespa</groupId> - <artifactId>security-utils</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> - - <!-- Test dependencies --> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-server</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <executions> - <execution> - <id>generate-simplified-vtag</id> - <phase>generate-sources</phase> - <goals> - <goal>exec</goal> - </goals> - <configuration> - <executable>src/main/bin/versiontagger.sh</executable> - <arguments> - <argument>${project.basedir}/../dist/vtag.map</argument> - <argument>${project.build.directory}/generated-sources/vtag/com/yahoo/vespa/http/client/core/Vtag.java</argument> - </arguments> - <sourceRoot>${project.build.directory}/generated-sources/vtag</sourceRoot> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <release>${vespaClients.jdk.releaseVersion}</release> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Xlint:-deprecation</arg> - <arg>-Xlint:-serial</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <finalName>${project.artifactId}-jar-with-dependencies</finalName> - <createDependencyReducedPom>false</createDependencyReducedPom> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>com.yahoo.vespa.http.client.runner.Runner</mainClass> - </transformer> - </transformers> - <shadeSourcesContent>true</shadeSourcesContent> - <!-- Let the user choose any apache lib version they want. --> - <relocations> - <relocation> - <pattern>org.apache.http</pattern> - <shadedPattern>com.yahoo.vespa.feeder.shaded.internal.apache.http</shadedPattern> - </relocation> - <relocation> - <pattern>org.apache.commons</pattern> - <shadedPattern>com.yahoo.vespa.feeder.shaded.internal.apache.commons</shadedPattern> - </relocation> - </relocations> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <executions> - <execution> - <id>attach-artifacts</id> - <phase>package</phase> - <goals> - <goal>attach-artifact</goal> - </goals> - <configuration> - <artifacts> - <artifact> - <file>target/${project.artifactId}-jar-with-dependencies.jar</file> - <type>jar</type> - <classifier>jar-with-dependencies</classifier> - </artifact> - </artifacts> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/vespa-http-client/src/main/bin/versiontagger.sh b/vespa-http-client/src/main/bin/versiontagger.sh deleted file mode 100755 index f4e4c23e50a..00000000000 --- a/vespa-http-client/src/main/bin/versiontagger.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -# Extracts the current version number (V_TAG_COMPONENT) from vtag.map and outputs this into a Java class. -# This replaces vespajlib/../VersionTagger.java as this module cannot depend on that, nor the dependencies -# of the resulting class. -# -# Author: bratseth - -source=$1 -destination=$2 -destinationDir=$(dirname $destination) - -mkdir -p $destinationDir - -versionNumber=$(cat $source | grep V_TAG_COMPONENT | awk '{print $2}' ) - -cat > $destination <<- END -package com.yahoo.vespa.http.client.core; - -public class Vtag { - public static final String V_TAG_COMPONENT = "$versionNumber"; -} -END diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java deleted file mode 100644 index d9ff09552ef..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.core.JsonReader; -import com.yahoo.vespa.http.client.core.XmlFeedReader; - -import java.io.InputStream; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * API for feeding document operations (add, removes or updates) to one or many Vespa clusters. - * Use the factory to configure and set up an instance of this. Instances are expensive - create one instance of this - * and use it for all feed operations (from multiple threads, if desired) for the duration of your client runtime. - * The feedclient does automatic error recovery and reconnects to hosts when connections die. - * - * A {@link FeedClientFactory} is provided to instantiate Sessions. - * - * See com.yahoo.text.Text.stripInvalidCharacters(String) to remove invalid characters from string fields before feeding - * - * Instances of this are multithread safe. - * - * @author dybis - * @see FeedClientFactory - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public interface FeedClient extends AutoCloseable { - - /** - * Issues a document operation to the configured cluster(s). - * If the pipeline and buffers are full, this call will be blocking, ensuring that operations are not - * produced faster than the can be handled. Transient failured are retried internally by this client. - * Exactly one callback will always be received for each (completed) call to this. - * - * @param documentId the document id of the document - * @param documentData the document data as JSON or XML (as specified when using the factory to create the API) - */ - default void stream(String documentId, CharSequence documentData) { - stream(documentId, documentData, null); - } - - /** - * Issues a document operation to the configured cluster(s). - * If the pipeline and buffers are full, this call will be blocking, ensuring that operations are not - * produced faster than the can be handled. Transient failured are retried internally by this client. - * Exactly one callback will always be received for each (completed) call to this. - * - * @param documentId the document id of the document - * @param documentData the document data as JSON or XML (as specified when using the factory to create the API) - * @param context a context object which will be accessible in the result of the callback, or null if none - */ - default void stream(String documentId, CharSequence documentData, Object context) { - stream(documentId, null, documentData, context); - } - - /** - * Issues a document operation to the configured cluster(s). - * If the pipeline and buffers are full, this call will be blocking, ensuring that operations are not - * produced faster than the can be handled. Transient failures are retried internally by this client. - * Exactly one callback will always be received for each (completed) call to this. - * - * @param documentId the document id of the document - * @param operationId the id to use for this operation, or null to let the client decide an operation id. - * This id must be unique for every operation. Passing the operation id allows clients - * to prepare to receive a response for it before issuing the operation to the client. - * @param documentData the document data as JSON or XML (as specified when using the factory to create the API) - * @param context a context object which will be accessible in the result of the callback, or null if none - */ - void stream(String documentId, String operationId, CharSequence documentData, Object context); - - /** - * Waits for all results to arrive and closes the FeedClient. Don't call any other method after calling close(). - * Does not throw any exceptions. - */ - @Override - void close(); - - /** - * Returns stats about the cluster - * - * @return JSON string with information about cluster - */ - String getStatsAsJson(); - - /** - * Utility function that takes an array of JSON documents and calls the FeedClient for each element. - * - * @param inputStream the stream to feed. This can be a very large stream. - * The outer element must be an array of document operations. - * @param feedClient the feed client that will receive the document operations - * @param numSent increased per document sent to API (but not waiting for results) - */ - static void feedJson(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { - JsonReader.read(inputStream, feedClient, numSent); - } - - /** - * Utility function that takes an array of XML documents and calls the FeedClient for each element. - * The XML document has to be formatted with line space on each line (like "regular" XML, but stricter - * than the specifications of XML). - * - * @param inputStream the stream to feed. This can be a very large stream. Operations must be enclosed in a - * top-level <vespafeed> tag - * @param feedClient the feed client that will receive the document operations - * @param numSent increased per document sent to API (but not waiting for results) - */ - static void feedXml(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { - try { - XmlFeedReader.read(inputStream, feedClient, numSent); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * This callback is executed when new results are arriving or an error occur. - * Don't do any heavy lifting in this thread (no IO, disk, or heavy CPU usage). - * This call back will run in a different thread than your main program so use e.g. - * AtomicInteger for counters and follow general guides for thread-safe programming. - * There is an example implementation in class SimpleLoggerResultCallback. - */ - interface ResultCallback { - - /** - * This callback is always called exactly once for each feed operation passed to the client - * instance, whether or not it was successful. - */ - void onCompletion(String docId, Result documentResult); - - /** - * Called with an exception whenever an endpoint specific error occurs during feeding. - * The error may or may not be transient - the operation will in both cases be retried until it's successful. - * This callback is intended for application level monitoring (logging, metrics, altering etc). - * Document specific errors will be reported back through {@link #onCompletion(String, Result)}. - * - * @see FeedEndpointException - * @param exception an exception specifying endpoint and cause. See {@link FeedEndpointException} for details. - */ - default void onEndpointException(FeedEndpointException exception) {} - - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java deleted file mode 100644 index ce2f9e0b140..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - - -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.api.FeedClientImpl; - -import java.time.Clock; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; - -/** - * Factory for creating FeedClient. - * - * @author dybis - * @deprecated Vespa-http-client will be removed in Vespa 8. It's Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class FeedClientFactory { - - /** - * Creates a FeedClient. Call this sparingly: Feed clients are expensive and should be as long-lived as possible. - * - * @param sessionParams parameters for connection, hosts, cluster configurations and more. - * @param resultCallback on each result, this callback is called. - * @return newly created FeedClient API object. - */ - public static FeedClient create(SessionParams sessionParams, FeedClient.ResultCallback resultCallback) { - return new FeedClientImpl(sessionParams, resultCallback, createTimeoutExecutor(), Clock.systemUTC()); - } - - static ScheduledThreadPoolExecutor createTimeoutExecutor() { - ScheduledThreadPoolExecutor timeoutExecutor; - timeoutExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("timeout-")); - timeoutExecutor.setRemoveOnCancelPolicy(true); - timeoutExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); - timeoutExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - return timeoutExecutor; - } - - private static class DaemonThreadFactory implements ThreadFactory { - - private final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); - private final String prefix; - - private DaemonThreadFactory(String prefix) { - this.prefix = prefix; - } - - @Override - public Thread newThread(Runnable runnable) { - Thread t = defaultThreadFactory.newThread(runnable); - t.setDaemon(true); - t.setName(prefix + t.getName()); - return t; - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedConnectException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedConnectException.java deleted file mode 100644 index 822a670ae08..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedConnectException.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Endpoint; - -/** - * An exception thrown when the client is unable to connect to a feed endpoint. - * - * @author bjorncs - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class FeedConnectException extends FeedEndpointException { - - public FeedConnectException(Throwable cause, Endpoint endpoint) { - super(createMessage(cause, endpoint), cause, endpoint); - } - - private static String createMessage(Throwable cause, Endpoint endpoint) { - return String.format("Handshake to endpoint '%s:%d' failed: %s", - endpoint.getHostname(), - endpoint.getPort(), - cause.getMessage()); - } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedEndpointException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedEndpointException.java deleted file mode 100644 index 304e2ea321b..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedEndpointException.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Endpoint; - -/** - * An exception type for endpoint specific errors. - * - * @see FeedConnectException - * @see FeedProtocolException - * @author bjorncs - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public abstract class FeedEndpointException extends RuntimeException { - - private final Endpoint endpoint; - - protected FeedEndpointException(String message, Throwable cause, Endpoint endpoint) { - super(message, cause); - this.endpoint = endpoint; - } - - public Endpoint getEndpoint() { - return endpoint; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedProtocolException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedProtocolException.java deleted file mode 100644 index 93041cced1c..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedProtocolException.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Endpoint; - -/** - * An exception thrown when a feed endpoint returns an error during feeding. - * - * @author bjorncs - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class FeedProtocolException extends FeedEndpointException { - - private final int httpStatusCode; - private final String httpResponseMessage; - - public FeedProtocolException(int httpStatusCode, - String httpResponseMessage, - Throwable cause, - Endpoint endpoint) { - super(createMessage(httpStatusCode, httpResponseMessage, endpoint), cause, endpoint); - this.httpStatusCode = httpStatusCode; - this.httpResponseMessage = httpResponseMessage; - } - - private static String createMessage(int httpStatusCode, - String httpResponseMessage, - Endpoint endpoint) { - return String.format("Endpoint '%s:%d' returned an error on handshake: %d - %s", - endpoint.getHostname(), - endpoint.getPort(), - httpStatusCode, - httpResponseMessage); - } - - public int getHttpStatusCode() { - return httpStatusCode; - } - - public String getHttpResponseMessage() { - return httpResponseMessage; - } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java deleted file mode 100644 index 5db592da02f..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Result.java +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.Exceptions; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -/** - * The result of a stream operation. A Result refers to a single document, - * but may contain more than one Result.Detail instances, as these pertains to a - * single endpoint, and a Result may wrap data for multiple endpoints. - * - * @author Einar M R Rosenvinge - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class Result { - - public enum ResultType { - OPERATION_EXECUTED, - TRANSITIVE_ERROR, - CONDITION_NOT_MET, - FATAL_ERROR - } - - private final Document document; - private final boolean success; - private final List<Detail> details; - private final String localTrace; - - public Result(Document document, Collection<Detail> values, StringBuilder localTrace) { - this.document = document; - this.details = Collections.unmodifiableList(new ArrayList<>(values)); - this.success = details.stream().allMatch(d -> d.getResultType() == ResultType.OPERATION_EXECUTED); - this.localTrace = localTrace == null ? null : localTrace.toString(); - } - - /** Returns the document id that this result is for */ - public String getDocumentId() { - return document.getDocumentId(); - } - - /** Returns the id of the operation this is the result of */ - public String getOperationId() { return document.getOperationId(); } - - /** Returns the document data */ - public CharSequence getDocumentDataAsCharSequence() { - return document.getDataAsString(); - } - - /** Returns the context of the object if any */ - public Object getContext() { - return document.getContext(); - } - - /** - * Returns true if the operation(s) was successful. If at least one {@link Detail} - * in {@link #getDetails()} is unsuccessful, this will return false. - */ - public boolean isSuccess() { - return success; - } - public boolean isSuccessOrConditionNotMet() { - return isSuccess() || - details.stream().allMatch(d -> d.getResultType() == Result.ResultType.OPERATION_EXECUTED || - d.getResultType() == Result.ResultType.CONDITION_NOT_MET); - } - - public List<Detail> getDetails() { return details; } - - /** Returns whether the operation has been set up with local tracing */ - public boolean hasLocalTrace() { - return localTrace != null; - } - - /** Information in a Result for a single operation sent to a single endpoint. */ - public static final class Detail { - - private final ResultType resultType; - private final Endpoint endpoint; - private final Exception exception; - private final String traceMessage; - - public Detail(Endpoint endpoint, ResultType resultType, String traceMessage, Exception e) { - this.endpoint = endpoint; - this.resultType = resultType; - this.exception = e; - this.traceMessage = traceMessage; - } - - public Detail(Endpoint endpoint) { - this.endpoint = endpoint; - this.resultType = ResultType.OPERATION_EXECUTED; - this.exception = null; - this.traceMessage = null; - } - - /** - * Returns the endpoint from which the result was received, - * or null if this failed before being assigned an endpoint - */ - public Endpoint getEndpoint() { - return endpoint; - } - - /** Returns whether the operation was successful */ - public boolean isSuccess() { - return resultType == ResultType.OPERATION_EXECUTED; - } - - /** Returns the result of the operation */ - public ResultType getResultType() { - return resultType; - } - - /** Returns any exception related to this Detail, if unsuccessful. Might be null. */ - public Exception getException() { - return exception; - } - - /** Returns any trace message produces, or null if none */ - public String getTraceMessage() { - return traceMessage; - } - - @Override - public String toString() { - StringBuilder b = new StringBuilder(); - b.append("Detail "); - b.append("resultType=").append(resultType); - if (exception != null) - b.append(" exception='").append(Exceptions.toMessageString(exception)).append("'"); - if (traceMessage != null && ! traceMessage.isEmpty()) - b.append(" trace='").append(traceMessage).append("'"); - if (endpoint != null) - b.append(" endpoint=").append(endpoint); - return b.toString(); - } - - } - - @Override - public String toString() { - return "Result for " + document + " " + (localTrace != null ? localTrace : ""); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java deleted file mode 100644 index 203f2d3b462..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.io.OutputStream; -import java.util.concurrent.BlockingQueue; - -/** - * A Session is an entity used to feed operations (like documents, removes or updates) to one Vespa - * cluster or several clusters in parallel. Current implementations are fail-fast, i.e. all feeding - * errors are propagated to the user as quickly as possible and with as much detail as possible. - * - * Implementations of this interface are required to be thread safe. - * - * A {@link SessionFactory} is provided to instantiate Sessions. - * - * @author Einar M R Rosenvinge - * @see SessionFactory - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public interface Session extends AutoCloseable { - - /** - * Returns an OutputStream that can be used to write ONE operation, identified by the - * given document ID. The data format must match the - * {@link com.yahoo.vespa.http.client.config.FeedParams.DataFormat} given when this - * Session was instantiated. Note that most data formats include the document ID in the - * actual buffer, which <em>must</em> match the document ID given as a parameter to this - * method. It is (as always) important to close the OutputStream returned - nothing - * is written to the wire until this is done. Note also that the Session holds a certain, - * dynamically determined maximum number of document operations in memory. - * When this threshold is reached, {@link java.io.OutputStream#close()} will block. - * - * - * @param documentId the unique ID identifying this operation in the system - * @return an OutputStream to write the operation payload into - */ - OutputStream stream(CharSequence documentId); - - /** - * Returns {@link Result}s for all operations enqueued by {@link #stream(CharSequence)}. - * Note that the order of results is non-deterministic, with <em>one</em> exception - results - * for one document ID are returned in the order they were enqueued. In all other cases - * Results may appear out-of-order. - * - * @return a blocking queue for retrieving results - * @see Result - */ - BlockingQueue<Result> results(); - - /** - * Closes this Session. All resources are freed, persistent connections are closed and - * internal threads are stopped. - * - * @throws RuntimeException in cases where underlying resources throw on shutdown/close - */ - void close(); - - /** - * Returns stats about the cluster. - * @return JSON string with information about cluster. - */ - String getStatsAsJson(); - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java deleted file mode 100644 index ef231ccc713..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; - -import java.time.Clock; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; - -/** - * Factory for creating {@link Session} instances. - * - * @author Einar M R Rosenvinge - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public final class SessionFactory { - - /** - * Creates a {@link Session} with the given parameters. - * - * @param params the parameters to use when creating the Session. - * @return a new Session instance - */ - public static Session create(SessionParams params) { - return createInternal(params); - } - - @SuppressWarnings("deprecation") - static Session createInternal(SessionParams params) { - return new com.yahoo.vespa.http.client.core.api.SessionImpl(params, createTimeoutExecutor(), Clock.systemUTC()); - } - - static ScheduledThreadPoolExecutor createTimeoutExecutor() { - ScheduledThreadPoolExecutor timeoutExecutor; - timeoutExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("timeout-")); - timeoutExecutor.setRemoveOnCancelPolicy(true); - timeoutExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); - timeoutExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); - return timeoutExecutor; - } - - /** - * Creates a {@link Session} to a single {@link Endpoint}, with default values for everything. - * For full control of all parameters, or to feed to more than one Endpoint or more than one {@link Cluster}, - * see {@link #create(com.yahoo.vespa.http.client.config.SessionParams)}. - * - * @param endpoint the Endpoint to feed to. - * @return a new Session instance - * @see #create(com.yahoo.vespa.http.client.config.SessionParams) - */ - public static Session create(Endpoint endpoint) { - return createInternal(endpoint); - } - - static Session createInternal(Endpoint endpoint) { - SessionParams params = new SessionParams.Builder().addCluster( - new Cluster.Builder().addEndpoint(endpoint).build()).build(); - return create(params); - } - - private static class DaemonThreadFactory implements ThreadFactory { - private final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); - private final String prefix; - - public DaemonThreadFactory(String prefix) { - this.prefix = prefix; - } - - @Override - public Thread newThread(Runnable runnable) { - Thread t = defaultThreadFactory.newThread(runnable); - t.setDaemon(true); - t.setName(prefix + t.getName()); - return t; - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java deleted file mode 100644 index e03bfd2b816..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallback.java +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.time.Duration; -import java.time.Instant; -import java.util.Date; -import java.util.Locale; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Simple implementation of the ResultCallback that logs to std err for every X documents: - * "Result received: 34 (1 failed so far, 2003 sent, success rate 1999.23 docs/sec)." - * On each failure it will print the Result object content. If tracing is enabled, it will print trace messages to - * std err as well. - * - * @author dybis - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class SimpleLoggerResultCallback implements FeedClient.ResultCallback { - - private final Object monitor = new Object(); - private int resultCounter = 0; - private int failureCounter = 0; - private final AtomicInteger sentDocumentCounter; - private final int printStatsForEveryXDocument; - private final boolean ignoreConditionNotMet; - private Instant startSampleInstant = Instant.now(); - private int startSampleResultCount = 0; - - protected void println(String output) { - System.err.println(output); - } - - /** - * Constructor - * - * @param sentDocumentCounter a counter that is increased outside this class, but can be nice to print here. - * @param printStatsForEveryXDocument how often to print stats. - */ - public SimpleLoggerResultCallback(AtomicInteger sentDocumentCounter, int printStatsForEveryXDocument, boolean ignoreConditionNotMet) { - this.sentDocumentCounter = sentDocumentCounter; - this.printStatsForEveryXDocument = printStatsForEveryXDocument; - this.ignoreConditionNotMet = ignoreConditionNotMet; - } - - /** - * Prints how many documents that are received, failed and sent. - */ - public void printProgress() { - synchronized (monitor) { - DocumentRate docRate = newSamplingPeriod(Instant.now()); - println(new Date() + " Result received: " + resultCounter - + " (" + failureCounter + " failed so far, " + sentDocumentCounter.get() - + " sent, success rate " + docRate + ")."); - } - } - - static class DocumentRate { - public final double rate; - DocumentRate(double rate) { - this.rate = rate; - } - @Override - public String toString() { - return String.format(Locale.US, "%.2f docs/sec", rate); - } - } - - /* - * Returns success results per second for last interval and resets variables. - */ - protected DocumentRate newSamplingPeriod(Instant now) { - double docsDelta = resultCounter - failureCounter - startSampleResultCount; - Duration duration = Duration.between(startSampleInstant, now); - startSampleInstant = now; - startSampleResultCount = resultCounter - failureCounter; - long durationMilliSecs = duration.toMillis() + 1; // +1 to avoid division by zero - return new DocumentRate(1000. * docsDelta / durationMilliSecs); - } - - int getResultCount() { - synchronized (monitor) { - return resultCounter; - } - } - - int getFailedDocumentCount() { - synchronized (monitor) { - return failureCounter; - } - } - - @Override - public void onCompletion(String docId, Result documentResult) { - synchronized (monitor) { - if (printStatsForEveryXDocument > 0 && (resultCounter % printStatsForEveryXDocument) == 0) { - printProgress(); - } - resultCounter++; - boolean success = ignoreConditionNotMet - ? documentResult.isSuccessOrConditionNotMet() - : documentResult.isSuccess(); - if ( ! success ) { - failureCounter++; - println("Failure: " + documentResult + (documentResult.getDetails().isEmpty() ? "" : ":")); - for (Result.Detail detail : documentResult.getDetails()) - println(" " + detail); - } - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java deleted file mode 100644 index 2fc63bbbcc5..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SyncFeedClient.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.SessionParams; - -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.ConcurrentModificationException; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.ThreadLocalRandom; - -/** - * A utility wrapper of a FeedClient which feeds a list of documents and blocks until all responses are returned, - * before returning the results. - * - * Not multithread safe: A sync feed client instance can only be used by a single thread - * (but it can and should be reused for multiple subsequent synchronous calls). - * - * @author bratseth - * @deprecated Vespa-http-client will be removed in Vespa 8. It's replaced by <a href="https://docs.vespa.ai/en/vespa-feed-client.html">vespa-feed-client</a> - */ -@Deprecated -public class SyncFeedClient implements AutoCloseable { - - private final FeedClient wrappedClient; - private final Callback callback; - - public SyncFeedClient(SessionParams params) { - callback = new SyncFeedClient.Callback(); - if (params.getFeedParams().getIdlePollFrequency() == null) { - params = params.toBuilder() - .setFeedParams(params.getFeedParams().toBuilder() - .setIdlePollFrequency(500.0) - .build()) - .build(); - } - this.wrappedClient = FeedClientFactory.create(params, callback); - } - - /** - * Calls FeedClient.stream for each entry in the list, blocks until all results are ready and returns them. - * This will block for at most the time it takes to feed these operations + clientTimeout given in the - * sessions params when creating this. - * - * @param operations the Vespa write operations to stream - * @return the result of feeding all these operations - */ - public SyncResult stream(List<SyncOperation> operations) { - callback.expectResultsOf(operations); - for (SyncOperation operation : operations) - wrappedClient.stream(operation.documentId, operation.operationId, operation.documentData, operation.context); - return callback.waitForResults(); - } - - @Override - public void close() { - wrappedClient.close(); - } - - /** Holds the arguments to a single stream operation */ - public static class SyncOperation { - - private final String documentId; - private final CharSequence documentData; - private final Object context; - - /** Operation id passed on to the Document created from this */ - private final String operationId; - - public SyncOperation(String documentId, CharSequence documentData) { - this(documentId, documentData, null); - } - - public SyncOperation(String documentId, CharSequence documentData, Object context) { - this(documentId, documentData, new BigInteger(64, ThreadLocalRandom.current()).toString(32), context); - } - - public SyncOperation(String documentId, CharSequence documentData, String operationId, Object context) { - this.documentId = Objects.requireNonNull(documentId, "documentId"); - this.documentData = Objects.requireNonNull(documentData, "documentData"); - this.context = context; - this.operationId = Objects.requireNonNull(operationId); - } - - } - - /** - * The result of a SyncFeedClient.stream call. This always holds exactly one Result per SyncOperation - * attempted, and the results are guaranteed to be returned in the same order as in the List of SyncOperations. - */ - public static class SyncResult { - - private final Exception exception; - private final List<Result> results; - - private SyncResult(List<Result> results, Exception exception) { - this.results = results; - this.exception = exception; - } - - /** - * Returns the results of this. This has the same size and order as the List of SyncOperations that - * created this. The list returned is modifiable and owned by the client. Multiple calls to this returns the - * same list instance. - */ - public List<Result> results() { return results; } - - /** - * Returns the last exception received when attempting the operations this is the result of, or null if none. - * Even if there is an exception, results() will return one Result per operation attempted. - */ - public Exception exception() { return exception; } - - /** Returns true if all Results in this are successful */ - public boolean isSuccess() { - return results.stream().allMatch(Result::isSuccess); - } - - } - - private static class Callback implements FeedClient.ResultCallback { - - private final Object monitor = new Object(); - - // The rest of the state of this is reset each time we call expectResultsOf - - private int resultsReceived; - private Exception exception = null; - - /** - * A map from operation ids to their results. This is initially populated with null values to keep track of - * which responses we are waiting for. - */ - private LinkedHashMap<String, Result> results = null; - - void expectResultsOf(List<SyncOperation> operations) { - synchronized (monitor) { - if (results != null) - throw new ConcurrentModificationException("A SyncFeedClient instance is used by multiple threads"); - - resultsReceived = 0; - exception = null; - results = new LinkedHashMap<>(operations.size()); - for (SyncOperation operation : operations) - results.put(operation.operationId, null); - } - } - - SyncResult waitForResults() { - try { - synchronized (monitor) { - while ( ! complete()) - monitor.wait(); - - SyncResult syncResult = new SyncResult(new ArrayList<>(results.values()), exception); - results = null; - return syncResult; - } - } - catch (InterruptedException e) { - throw new RuntimeException("Interrupted while waiting for feeding results", e); - } - } - - @Override - public void onCompletion(String docId, Result documentResult) { - synchronized (monitor) { - if ( ! results.containsKey(documentResult.getOperationId())) return; // Stale result - ignore - - Result previousValue = results.put(documentResult.getOperationId(), documentResult); - if (previousValue != null) - throw new IllegalStateException("Received duplicate result for " + docId); - - resultsReceived++; - if (complete()) - monitor.notifyAll(); - } - } - - @Override - public void onEndpointException(FeedEndpointException exception) { - this.exception = exception; // We will still receive one onCompletion per stream invocation done - } - - private boolean complete() { - return resultsReceived == results.size(); - } - - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java deleted file mode 100644 index 4f72c380e59..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Cluster.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -/** - * A set of {@link Endpoint} instances. Construct using {@link Cluster.Builder}. - * - * @author Einar M R Rosenvinge - */ -public final class Cluster { - - /** Builder for {@link Cluster}. */ - public final static class Builder { - private final List<Endpoint> endpoints = new LinkedList<>(); - private String route = null; - - /** - * Adds an Endpoint (a HTTP gateway) to this Cluster. - * - * @param endpoint the Endpoint to add - * @return this, for chaining - */ - public Builder addEndpoint(Endpoint endpoint) { - endpoints.add(endpoint); - return this; - } - - /** - * Sets a route specific to this cluster, which overrides the route set in {@link com.yahoo.vespa.http.client.config.FeedParams#getRoute()}. - * - * @param route a route specific to this cluster - * @return this, for chaining - */ - public Builder setRoute(String route) { - this.route = route; - return this; - } - - public Cluster build() { - return new Cluster(endpoints, route); - } - - public String getRoute() { - return route; - } - } - private final List<Endpoint> endpoints; - private final String route; - - private Cluster(List<Endpoint> endpoints, String route) { - this.endpoints = Collections.unmodifiableList(new ArrayList<>(endpoints)); - this.route = route; - } - - public List<Endpoint> getEndpoints() { - return endpoints; - } - - public String getRoute() { - return route; - } - - @Override - public String toString() { - return "cluster with endpoints " + endpoints + " and route '" + route + "'"; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java deleted file mode 100644 index 00cc2512ae3..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java +++ /dev/null @@ -1,486 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; -import java.nio.file.Path; -import java.time.Duration; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -/** - * Connection level parameters. - * This class is immutable - * and has no public constructor - to instantiate one, use a {@link Builder}. - * - * @author Einar M R Rosenvinge - */ -public final class ConnectionParams { - - /** - * Builder for {@link ConnectionParams}. - */ - public static final class Builder { - - private SSLContext sslContext = null; - private HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier(); - private final Multimap<String, String> headers = ArrayListMultimap.create(); - private final Map<String, HeaderProvider> headerProviders = new HashMap<>(); - private int numPersistentConnectionsPerEndpoint = 1; - private String proxyHost = null; - private int proxyPort = 8080; - private boolean useCompression = false; - private int maxRetries = 100; - private long minTimeBetweenRetriesMs = 700; - private boolean dryRun = false; - private boolean runThreads = true; - private int traceLevel = 0; - private int traceEveryXOperation = 0; - private boolean printTraceToStdErr = true; - private boolean useTlsConfigFromEnvironment = false; - private Duration connectionTimeToLive = Duration.ofSeconds(30); - private Path privateKey; - private Path certificate; - private Path caCertificates; - - /** - * Use TLS configuration through the standard Vespa environment variables. - * Setting this to 'true' will override any other TLS/HTTPS related configuration. - */ - public Builder setUseTlsConfigFromEnvironment(boolean useTlsConfigFromEnvironment) { - this.useTlsConfigFromEnvironment = useTlsConfigFromEnvironment; - return this; - } - - /** - * Sets the SSLContext for the connection to the gateway when SSL is enabled for Endpoint. - * Default null (no ssl). See also Endpoint configuration. - * - * @param sslContext sslContext for connection to gateway. - * @return pointer to builder. - */ - public Builder setSslContext(SSLContext sslContext) { - this.sslContext = sslContext; - return this; - } - - /** - * Sets the {@link HostnameVerifier} for the connection to the gateway when SSL is enabled for Endpoint. - * Defaults to instance returned by {@link SSLConnectionSocketFactory#getDefaultHostnameVerifier()}. - * - * @param hostnameVerifier hostname verifier for connection to gateway. - * @return pointer to builder. - */ - public Builder setHostnameVerifier(HostnameVerifier hostnameVerifier) { - this.hostnameVerifier = hostnameVerifier; - return this; - } - - /** - * Set path to private key and certificate files. Both the private key and certificate must be PEM-encoded. - */ - public Builder setCertificateAndPrivateKey(Path privateKey, Path certificate) { - this.privateKey = privateKey; - this.certificate = certificate; - return this; - } - - /** - * Set path a PEM file containing the CA certificates. - */ - public Builder setCaCertificates(Path caCertificates) { - this.caCertificates = caCertificates; - return this; - } - - /** - * Set custom headers to be used - * - * @param key header name - * @param value header value - * @return pointer to builder. - */ - public Builder addHeader(String key, String value) { - headers.put(key, value); - return this; - } - - /** - * Adds a header provider for dynamic headers; headers where the value may change during a feeding session - * (e.g. security tokens with limited life time). Only one {@link HeaderProvider} is allowed for a given header name. - * - * @param provider A provider for a dynamic header - * @return pointer to builder. - * @throws IllegalArgumentException if a provider is already registered for the given header name - */ - public Builder addDynamicHeader(String headerName, HeaderProvider provider) { - Objects.requireNonNull(headerName, "Header name cannot be null"); - Objects.requireNonNull(provider, "Header provider cannot be null"); - if (headerProviders.containsKey(headerName)) { - throw new IllegalArgumentException("Provider already registered for name '" + headerName + "'"); - } - headerProviders.put(headerName, provider); - return this; - } - - /** - * The number of connections between the http client and the gateways. A very low number can result - * in the network not fully utilized and the round-trip time can be a limiting factor. A low number - * can cause skew in distribution of load between gateways. A too high number will cause - * many threads to run, more context switching and potential more memory usage. We recommend using about - * 16 connections per gateway. - * - * @param numPersistentConnectionsPerEndpoint number of channels per endpoint - * @return pointer to builder. - */ - public Builder setNumPersistentConnectionsPerEndpoint(int numPersistentConnectionsPerEndpoint) { - this.numPersistentConnectionsPerEndpoint = numPersistentConnectionsPerEndpoint; - return this; - } - - /** - * Sets the HTTP proxy host name to use. - * - * @param proxyHost host name for proxy. - * @return pointer to builder. - */ - public Builder setProxyHost(String proxyHost) { - this.proxyHost = proxyHost; - return this; - } - - /** - * Sets the HTTP proxy host port to use. - * - * @param proxyPort host port for proxy. - * @return pointer to builder. - */ - public Builder setProxyPort(int proxyPort) { - this.proxyPort = proxyPort; - return this; - } - - /** - * Set whether compression of document operations during communication to server should be enabled. - * - * @param useCompression true if compression should be enabled. - * @return pointer to builder. - */ - public Builder setUseCompression(boolean useCompression) { - this.useCompression = useCompression; - return this; - } - - /** - * Set how many times to retry sending an operation to a gateway when encountering transient problems. - * - * @param maxRetries max number of retries - * @return pointer to builder. - */ - public Builder setMaxRetries(int maxRetries) { - this.maxRetries = maxRetries; - return this; - } - - /** - * Set to true to skip making network connections and instead - * let requests complete successfully with no effect. - */ - public Builder setDryRun(boolean dryRun) { - this.dryRun = dryRun; - return this; - } - - /** - * Set to false to skip starting io threads, such that any operation must be driven by a calling thread. - * Useful for testing. - */ - public Builder setRunThreads(boolean runThreads) { - this.runThreads = runThreads; - return this; - } - - /** - * Set the min time between retries when temporarily failing against a gateway. - * - * @param minTimeBetweenRetries the min time value - * @param unit the unit of the min time. - * @return pointer to builder. - */ - public Builder setMinTimeBetweenRetries(long minTimeBetweenRetries, TimeUnit unit) { - this.minTimeBetweenRetriesMs = unit.toMillis(minTimeBetweenRetries); - return this; - } - - public long getMinTimeBetweenRetriesMs() { - return minTimeBetweenRetriesMs; - } - - /** - * Sets the trace level for tracing messagebus. 0 means to tracing. - * - * @param traceLevel tracelevel, larger value means more tracing. - * @return pointer to builder. - */ - public Builder setTraceLevel(int traceLevel) { - this.traceLevel = traceLevel; - return this; - } - - /** - * How often to trace messages in client. Please note that this does not affect tracing with messagebus - * - * @param traceEveryXOperation if zero, no tracing, 1 = every message, and so on. - * @return pointer to builder. - */ - public Builder setTraceEveryXOperation(int traceEveryXOperation) { - this.traceEveryXOperation = traceEveryXOperation; - return this; - } - - /** - * If enabled will write internal trace to stderr. - * - * @param printTraceToStdErr if value is true it is enabled. - * @return pointer to builder. - */ - public Builder setPrintTraceToStdErr(boolean printTraceToStdErr) { - this.printTraceToStdErr = printTraceToStdErr; - return this; - } - - /** - * Set the maximum time to live for persistent connections - */ - public Builder setConnectionTimeToLive(Duration connectionTimeToLive) { - this.connectionTimeToLive = connectionTimeToLive; - return this; - } - - public ConnectionParams build() { - return new ConnectionParams( - sslContext, - privateKey, - certificate, - caCertificates, - hostnameVerifier, - headers, - headerProviders, - numPersistentConnectionsPerEndpoint, - proxyHost, - proxyPort, - useCompression, - maxRetries, - minTimeBetweenRetriesMs, - dryRun, - runThreads, - traceLevel, - traceEveryXOperation, - printTraceToStdErr, - useTlsConfigFromEnvironment, - connectionTimeToLive); - } - - public int getNumPersistentConnectionsPerEndpoint() { - return numPersistentConnectionsPerEndpoint; - } - - public String getProxyHost() { - return proxyHost; - } - - public boolean isDryRun() { - return dryRun; - } - - public boolean runThreads() { return runThreads; } - - public int getMaxRetries() { - return maxRetries; - } - public int getTraceLevel() { - return traceLevel; - } - public int getTraceEveryXOperation() { - return traceEveryXOperation; - } - - public boolean getPrintTraceToStdErr() { - return printTraceToStdErr; - } - - public int getProxyPort() { - return proxyPort; - } - - public SSLContext getSslContext() { - return sslContext; - } - - public HostnameVerifier getHostnameVerifier() { - return hostnameVerifier; - } - - public boolean useTlsConfigFromEnvironment() { - return useTlsConfigFromEnvironment; - } - - public Duration getConnectionTimeToLive() { - return connectionTimeToLive; - } - public Path getPrivateKey() { return privateKey; } - public Path getCertificate() { return certificate; } - public Path getCaCertificates() { return caCertificates; } - } - - private final SSLContext sslContext; - private final Path privateKey; - private final Path certificate; - private final Path caCertificates; - private final HostnameVerifier hostnameVerifier; - private final Multimap<String, String> headers = ArrayListMultimap.create(); - private final Map<String, HeaderProvider> headerProviders = new HashMap<>(); - private final int numPersistentConnectionsPerEndpoint; - private final String proxyHost; - private final int proxyPort; - private final boolean useCompression; - private final int maxRetries; - private final long minTimeBetweenRetriesMs; - private final boolean dryRun; - private final boolean runThreads; - private final int traceLevel; - private final int traceEveryXOperation; - private final boolean printTraceToStdErr; - private final boolean useTlsConfigFromEnvironment; - private final Duration connectionTimeToLive; - - private ConnectionParams( - SSLContext sslContext, - Path privateKey, Path certificate, Path caCertificates, - HostnameVerifier hostnameVerifier, - Multimap<String, String> headers, - Map<String, HeaderProvider> headerProviders, - int numPersistentConnectionsPerEndpoint, - String proxyHost, - int proxyPort, - boolean useCompression, - int maxRetries, - long minTimeBetweenRetriesMs, - boolean dryRun, - boolean runThreads, - int traceLevel, - int traceEveryXOperation, - boolean printTraceToStdErr, - boolean useTlsConfigFromEnvironment, - Duration connectionTimeToLive) { - this.sslContext = sslContext; - this.privateKey = privateKey; - this.certificate = certificate; - this.caCertificates = caCertificates; - this.hostnameVerifier = hostnameVerifier; - this.useTlsConfigFromEnvironment = useTlsConfigFromEnvironment; - this.connectionTimeToLive = connectionTimeToLive; - this.headers.putAll(headers); - this.headerProviders.putAll(headerProviders); - this.numPersistentConnectionsPerEndpoint = numPersistentConnectionsPerEndpoint; - this.proxyHost = proxyHost; - this.proxyPort = proxyPort; - this.useCompression = useCompression; - this.maxRetries = maxRetries; - this.minTimeBetweenRetriesMs = minTimeBetweenRetriesMs; - this.dryRun = dryRun; - this.runThreads = runThreads; - this.traceLevel = traceLevel; - this.traceEveryXOperation = traceEveryXOperation; - this.printTraceToStdErr = printTraceToStdErr; - } - - @JsonIgnore - public SSLContext getSslContext() { - return sslContext; - } - - @JsonIgnore - public HostnameVerifier getHostnameVerifier() { - return hostnameVerifier; - } - - public Collection<Map.Entry<String, String>> getHeaders() { - return Collections.unmodifiableCollection(headers.entries()); - } - @JsonIgnore - public Map<String, HeaderProvider> getDynamicHeaders() { - return Collections.unmodifiableMap(headerProviders); - } - - public int getNumPersistentConnectionsPerEndpoint() { - return numPersistentConnectionsPerEndpoint; - } - - public String getProxyHost() { - return proxyHost; - } - - public int getProxyPort() { - return proxyPort; - } - - public boolean getUseCompression() { - return useCompression; - } - - public int getMaxRetries() { - return maxRetries; - } - - public long getMinTimeBetweenRetriesMs() { - return minTimeBetweenRetriesMs; - } - - public boolean isDryRun() { - return dryRun; - } - - public boolean runThreads() { return runThreads; } - - public int getTraceLevel() { - return traceLevel; - } - - public int getTraceEveryXOperation() { - return traceEveryXOperation; - } - - public boolean getPrintTraceToStdErr() { - return printTraceToStdErr; - } - - public boolean useTlsConfigFromEnvironment() { - return useTlsConfigFromEnvironment; - } - - public Duration getConnectionTimeToLive() { - return connectionTimeToLive; - } - - /** - * A header provider that provides a header value. {@link #getHeaderValue()} is called each time a new HTTP request - * is constructed by {@link com.yahoo.vespa.http.client.FeedClient}. - * - * Important: The implementation of {@link #getHeaderValue()} must be thread-safe! - */ - public interface HeaderProvider { String getHeaderValue(); } - - public Path getPrivateKey() { return privateKey; } - public Path getCertificate() { return certificate; } - public Path getCaCertificates() { return caCertificates; } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java deleted file mode 100644 index ae0cf19a3d1..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/Endpoint.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import java.io.Serializable; -import java.net.URL; - -/** - * Represents an endpoint, in most cases a JDisc container - * in a Vespa cluster configured with <code>document-api</code>. - * - * @author Einar M R Rosenvinge - */ -public final class Endpoint implements Serializable { - - private static final int DEFAULT_PORT = 4080; - - private final String hostname; - private final int port; - private final boolean useSsl; - - private Endpoint(String hostname, int port, boolean useSsl) { - if (hostname.startsWith("https://")) { - throw new RuntimeException("Hostname should be name of machine, not prefixed with protocol (https://)"); - } - // A lot of people put http:// before the servername, let us allow that. - if (hostname.startsWith("http://")) { - this.hostname = hostname.replaceFirst("http://", ""); - } else { - this.hostname = hostname; - } - this.port = port; - this.useSsl = useSsl; - } - - public String getHostname() { - return hostname; - } - - public int getPort() { - return port; - } - - public boolean isUseSsl() { - return useSsl; - } - - @Override - public String toString() { - return hostname + ":" + port + " ssl=" + useSsl; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof Endpoint)) return false; - Endpoint endpoint = (Endpoint) o; - return port == endpoint.port && useSsl == endpoint.useSsl && hostname.equals(endpoint.hostname); - } - - @Override - public int hashCode() { - int result = hostname.hashCode(); - result = 31 * result + port; - result = 31 * result + (useSsl ? 1 : 0); - return result; - } - - /** Creates an Endpoint with the default port and without using SSL */ - public static Endpoint create(String hostname) { - return new Endpoint(hostname, DEFAULT_PORT, false); - } - - /** Creates an Endpoint with the given hostname, port and SSL setting. */ - public static Endpoint create(String hostname, int port, boolean useSsl) { - return new Endpoint(hostname, port, useSsl); - } - - public static Endpoint create(URL url) { - return new Endpoint(url.getHost(), url.getPort(), "https".equals(url.getProtocol())); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java deleted file mode 100644 index 01f314a7e36..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import java.util.concurrent.TimeUnit; - -/** - * Feed level parameters. This class is immutable - * and has no public constructor - to instantiate one, use a {@link Builder}. - - * @author Einar M R Rosenvinge - */ -public final class FeedParams { - - public boolean getDenyIfBusyV3() { return denyIfBusyV3; } - - public long getMaxSleepTimeMs() { return maxSleepTimeMs; } - - public boolean getSilentUpgrade() { return silentUpgrade; } - - /** - * Enumeration of data formats that are acceptable by the - * {@link com.yahoo.vespa.http.client.FeedClient} methods. - */ - public enum DataFormat { - /** UTF-8-encoded XML. Preamble is not necessary. */ - XML_UTF8, - JSON_UTF8 - } - /** - * Mutable class used to instantiate a {@link FeedParams}. - */ - public static final class Builder { - - private DataFormat dataFormat = DataFormat.JSON_UTF8; - private long serverTimeout = TimeUnit.SECONDS.toMillis(180); - private long clientTimeout = TimeUnit.SECONDS.toMillis(20); - private String route = null; - private int maxChunkSizeBytes = 50 * 1024; - private int maxInFlightRequests = 5000; - private long localQueueTimeOut = 180000; - private String priority = null; - private boolean denyIfBusyV3 = true; - private long maxSleepTimeMs = 3000; - private boolean silentUpgrade = true; - private Double idlePollFrequency = null; - - /** - * Make server not throw 4xx/5xx for situations that are normal during upgrade as this can esily mask - * other problems. This feature need to be supported on server side to work, but it is still safe - * to enable it, even if server does not yet support it. As of Nov 22 2016 it is not yet implemented on - * the server side. - * @param silentUpgrade true for reducing "false" 4xx/5xx. - * @return this, for chaining - */ - public Builder setSilentUpgrade(boolean silentUpgrade) { - this.silentUpgrade = silentUpgrade; - return this; - } - - /** - * When throttling the load due to transient errors on gateway, what is the most time to wait between - * requests per thread. Only active for V3 protocol. - * @param ms max with time - * @return this, for chaining - */ - public Builder setMaxSleepTimeMs(long ms) { - this.maxSleepTimeMs = ms; - return this; - } - - /** - * If this is set to false, the gateway will block threads until messagebus can send the message. - * If true, the gateway will exit and fail the request early if there are many threads already - * blocked. - * @param value true to reduce number of blocked threads in gateway. - * @return this, for chaining - */ - public Builder setDenyIfBusyV3(boolean value) { - denyIfBusyV3 = value; - return this; - } - - /** - * Sets the data format to be used. - * - * @param dataFormat the data format to be used. - * @see DataFormat - * @return this, for chaining - */ - public Builder setDataFormat(DataFormat dataFormat) { - this.dataFormat = dataFormat; - return this; - } - - /** - * Sets a route to be used for all Clusters, unless overridden on a per-cluster basis - * in {@link com.yahoo.vespa.http.client.config.Cluster#getRoute()}. - * - * @param route a route to be used for all Clusters. - * @return this, for chaining - */ - public Builder setRoute(String route) { - this.route = route; - return this; - } - - /** - * Sets the server-side timeout of each operation - i.e. the timeout used by - * the server endpoint for operations going over the message bus protocol into - * Vespa. - * - * Note that the TOTAL timeout of any one operation in this API would be - * {@link #getServerTimeout(java.util.concurrent.TimeUnit)} + - * {@link #getClientTimeout(java.util.concurrent.TimeUnit)}. - * - * @param serverTimeout timeout value - * @param unit unit of timeout value - * @return this, for chaining - */ - public Builder setServerTimeout(long serverTimeout, TimeUnit unit) { - if (serverTimeout <= 0L) { - throw new IllegalArgumentException("Server timeout cannot be zero or negative."); - } - this.serverTimeout = unit.toMillis(serverTimeout); - return this; - } - - /** - * Sets the client-side timeout for each operation. If BOTH the server-side - * timeout AND this timeout has passed, the {@link com.yahoo.vespa.http.client.FeedClient} - * will synthesize a {@link com.yahoo.vespa.http.client.Result}. - * - * Note that the TOTAL timeout of any one operation in this API would be - * {@link #getServerTimeout(java.util.concurrent.TimeUnit)} + - * {@link #getClientTimeout(java.util.concurrent.TimeUnit)}, - * after which a result callback is guaranteed to be made. - * - * @param clientTimeout timeout value - * @param unit unit of timeout value - * @return this, for chaining - */ - public Builder setClientTimeout(long clientTimeout, TimeUnit unit) { - if (clientTimeout <= 0L) { - throw new IllegalArgumentException("Client timeout cannot be zero or negative."); - } - this.clientTimeout = unit.toMillis(clientTimeout); - return this; - } - - /** - * Sets the maximum number of bytes of document data to send per HTTP request. - * - * @param maxChunkSizeBytes max number of bytes per HTTP request. - * @return this, for chaining - */ - public Builder setMaxChunkSizeBytes(int maxChunkSizeBytes) { - this.maxChunkSizeBytes = maxChunkSizeBytes; - return this; - } - - /** - * Sets the maximum number of operations to be in-flight. - * - * @param maxInFlightRequests max number of operations. - * @return this, for chaining - */ - public Builder setMaxInFlightRequests(int maxInFlightRequests) { - this.maxInFlightRequests = maxInFlightRequests; - return this; - } - - /** - * Sets the number of milliseconds until we respond with a timeout for a document operation - * if we still have not received a response. - */ - public Builder setLocalQueueTimeOut(long timeOutMs) { - this.localQueueTimeOut = timeOutMs; - return this; - } - - /** - * Set what frequency to poll for async responses. Default is 10hz (every 0.1s), but 1000hz when using SyncFeedClient - */ - public Builder setIdlePollFrequency(Double idlePollFrequency) { - this.idlePollFrequency = idlePollFrequency; - return this; - } - - /** - * Sets the messagebus priority. The allowed values are HIGHEST, VERY_HIGH, HIGH_[1-3], - * NORMAL_[1-6], LOW_[1-3], VERY_LOW, and LOWEST.. - * @param priority messagebus priority of this message. - * @return this, for chaining - */ - public Builder setPriority(String priority) { - if (priority == null) { - return this; - } - switch (priority) { - case "HIGHEST": - case "VERY_HIGH": - case "HIGH_1": - case "HIGH_2": - case "HIGH_3": - case "NORMAL_1": - case "NORMAL_2": - case "NORMAL_3": - case "NORMAL_4": - case "NORMAL_5": - case "NORMAL_6": - case "LOW_1": - case "LOW_2": - case "LOW_3": - case "VERY_LOW": - case "LOWEST": - this.priority = priority; - return this; - default: - throw new IllegalArgumentException("Unknown value for priority: " + priority - + " Allowed values are HIGHEST, VERY_HIGH, HIGH_[1-3], " + - "NORMAL_[1-6], LOW_[1-3], VERY_LOW, and LOWEST."); - } - } - - /** - * Instantiates a {@link FeedParams}. - * - * @return a FeedParams object with the parameters of this Builder - */ - public FeedParams build() { - return new FeedParams( - dataFormat, serverTimeout, clientTimeout, route, - maxChunkSizeBytes, maxInFlightRequests, localQueueTimeOut, priority, - denyIfBusyV3, maxSleepTimeMs, silentUpgrade, idlePollFrequency); - } - - public long getClientTimeout(TimeUnit unit) { - return unit.convert(clientTimeout, TimeUnit.MILLISECONDS); - } - - public long getServerTimeout(TimeUnit unit) { - return unit.convert(serverTimeout, TimeUnit.MILLISECONDS); - } - - public String getRoute() { - return route; - } - - public DataFormat getDataFormat() { - return dataFormat; - } - - public int getMaxChunkSizeBytes() { - return maxChunkSizeBytes; - } - - public int getMaxInFlightRequests() { - return maxInFlightRequests; - } - - } - - // NOTE! See toBuilder at the end of this class if you add fields here - - private final DataFormat dataFormat; - private final long serverTimeoutMillis; - private final long clientTimeoutMillis; - private final String route; - private final int maxChunkSizeBytes; - private final int maxInFlightRequests; - private final long localQueueTimeOut; - private final String priority; - private final boolean denyIfBusyV3; - private final long maxSleepTimeMs; - private final boolean silentUpgrade; - private final Double idlePollFrequency; - - private FeedParams(DataFormat dataFormat, long serverTimeout, long clientTimeout, String route, - int maxChunkSizeBytes, final int maxInFlightRequests, - long localQueueTimeOut, String priority, boolean denyIfBusyV3, long maxSleepTimeMs, - boolean silentUpgrade, Double idlePollFrequency) { - this.dataFormat = dataFormat; - this.serverTimeoutMillis = serverTimeout; - this.clientTimeoutMillis = clientTimeout; - this.route = route; - this.maxChunkSizeBytes = maxChunkSizeBytes; - this.maxInFlightRequests = maxInFlightRequests; - this.localQueueTimeOut = localQueueTimeOut; - this.priority = priority; - this.denyIfBusyV3 = denyIfBusyV3; - this.maxSleepTimeMs = maxSleepTimeMs; - this.silentUpgrade = silentUpgrade; - this.idlePollFrequency = idlePollFrequency; - - } - - public DataFormat getDataFormat() { return dataFormat; } - public String getRoute() { return route; } - public long getServerTimeout(TimeUnit unit) { return unit.convert(serverTimeoutMillis, TimeUnit.MILLISECONDS); } - public long getClientTimeout(TimeUnit unit) { return unit.convert(clientTimeoutMillis, TimeUnit.MILLISECONDS); } - - public int getMaxChunkSizeBytes() { return maxChunkSizeBytes; } - public String getPriority() { return priority; } - - public String toUriParameters() { - StringBuilder b = new StringBuilder(); - b.append("&dataformat=").append(dataFormat.name()); //name in dataFormat enum obviously must be ascii - return b.toString(); - } - - public int getMaxInFlightRequests() { return maxInFlightRequests; } - public long getLocalQueueTimeOut() { return localQueueTimeOut; } - public Double getIdlePollFrequency() { return idlePollFrequency; } - - /** Returns a builder initialized to the values of this */ - public FeedParams.Builder toBuilder() { - Builder b = new Builder(); - b.setDataFormat(dataFormat); - b.setServerTimeout(serverTimeoutMillis, TimeUnit.MILLISECONDS); - b.setClientTimeout(clientTimeoutMillis, TimeUnit.MILLISECONDS); - b.setRoute(route); - b.setMaxChunkSizeBytes(maxChunkSizeBytes); - b.setMaxInFlightRequests(maxInFlightRequests); - b.setPriority(priority); - b.setDenyIfBusyV3(denyIfBusyV3); - b.setMaxSleepTimeMs(maxSleepTimeMs); - b.setSilentUpgrade(silentUpgrade); - b.setIdlePollFrequency(idlePollFrequency); - return b; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java deleted file mode 100644 index e8052fa7faa..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -/** - * Parameters given to a {@link com.yahoo.vespa.http.client.FeedClientFactory} - * when creating {@link com.yahoo.vespa.http.client.FeedClient}s. This class is immutable - * and has no public constructor - to instantiate one, use a {@link Builder}. - * - * @author Einar M R Rosenvinge - * @see com.yahoo.vespa.http.client.FeedClientFactory - * @see Builder - */ -public final class SessionParams { - - /** - * Interface for handling serious errors with connection. - */ - public interface ErrorReporter { - void onSessionError(Endpoint endpoint, String oldSessionID, String newSessionId); - } - - /** - * Mutable class used to instantiate a {@link SessionParams}. A builder - * instance will at the very least contain cluster settings ( - * {@link #addCluster(Cluster)}), for supporting SSL and similar transport - * settings, use {@link #setConnectionParams(ConnectionParams)}. - */ - public static final class Builder { - - private final List<Cluster> clusters = new LinkedList<>(); - private FeedParams feedParams = new FeedParams.Builder().build(); - private ConnectionParams connectionParams = new ConnectionParams.Builder().build(); - private int clientQueueSize = 10000; - private ErrorReporter errorReporter = null; - private int throttlerMinSize = 0; - - /** - * Add a Vespa installation for feeding documents into. - * - * @return this Builder instance, to support chaining - */ - public Builder addCluster(Cluster cluster) { - clusters.add(cluster); - return this; - } - - /** - * Set parameters used for feeding the documents in the receiving - * cluster. Reasonable defaults are supplied, so setting this should not - * be necessary for testing. - * - * @return this builder instance to support chaining - */ - public Builder setFeedParams(FeedParams feedParams) { - this.feedParams = feedParams; - return this; - } - - /** - * Transport parameters, like custom HTTP headers. - * - * @return this Builder instance, to support chaining - */ - public Builder setConnectionParams(ConnectionParams connectionParams) { - this.connectionParams = connectionParams; - return this; - } - - /** - * Sets an error reporter that is invoked in case of serious errors. - * - * @param errorReporter the handler - * @return pointer to builder. - */ - public Builder setErrorReporter(ErrorReporter errorReporter) { - this.errorReporter = errorReporter; - return this; - } - - /** - * Sets the maximum number of document operations to hold in memory, waiting to be - * sent to Vespa. When this threshold is reached, {@link java.io.OutputStream#close()} will block. - * - * @param clientQueueSize the maximum number of document operations to hold in memory. - * @return pointer to builder. - */ - public Builder setClientQueueSize(int clientQueueSize) { - this.clientQueueSize = clientQueueSize; - return this; - } - - /** - * Sets the minimum queue size of the throttler. If this is zero, it means that dynamic throttling is - * not enabled. Otherwise it is the minimum size of the throttler for how many parallel requests that are - * accepted. The max size of the throttler is the clientQueueSize. - * @return the minimum number of requests to be used in throttler or zero if throttler is static. - * - * @param throttlerMinSize the value of the min size. - */ - public Builder setThrottlerMinSize(int throttlerMinSize) { - this.throttlerMinSize = throttlerMinSize; - return this; - } - - /** - * Instantiates a {@link SessionParams} that can be given to a {@link com.yahoo.vespa.http.client.FeedClientFactory}. - * - * @return a SessionParams object with the parameters of this Builder - */ - public SessionParams build() { - return new SessionParams( - clusters, feedParams, connectionParams, clientQueueSize, errorReporter, throttlerMinSize); - } - - public FeedParams getFeedParams() { - return feedParams; - } - public ConnectionParams getConnectionParams() { - return connectionParams; - } - public int getClientQueueSize() { - return clientQueueSize; - } - public int getThrottlerMinSize() { - return throttlerMinSize; - } - } - - // NOTE! See toBuilder at the end of this class if you add fields here - - private final List<Cluster> clusters; - private final FeedParams feedParams; - private final ConnectionParams connectionParams; - private final int clientQueueSize; - private final ErrorReporter errorReport; - private final int throttlerMinSize; - - private SessionParams(Collection<Cluster> clusters, - FeedParams feedParams, - ConnectionParams connectionParams, - int clientQueueSize, - ErrorReporter errorReporter, - int throttlerMinSize) { - this.clusters = Collections.unmodifiableList(new ArrayList<>(clusters)); - this.feedParams = feedParams; - this.connectionParams = connectionParams; - this.clientQueueSize = clientQueueSize; - this.errorReport = errorReporter; - this.throttlerMinSize = throttlerMinSize; - } - - public List<Cluster> getClusters() { - return clusters; - } - - public FeedParams getFeedParams() { - return feedParams; - } - - public ConnectionParams getConnectionParams() { - return connectionParams; - } - - public int getClientQueueSize() { - return clientQueueSize; - } - - public int getThrottlerMinSize() { - return throttlerMinSize; - } - - public ErrorReporter getErrorReport() { - return errorReport; - } - - public Builder toBuilder() { - Builder b = new Builder(); - clusters.forEach(c -> b.addCluster(c)); - b.setFeedParams(feedParams); - b.setConnectionParams(connectionParams); - b.setClientQueueSize(clientQueueSize); - b.setErrorReporter(errorReport); - b.setThrottlerMinSize(throttlerMinSize); - return b; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/package-info.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/package-info.java deleted file mode 100644 index 9c07e819e90..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/package-info.java +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Settings for creating clients/sessions. - * - * NOTE: This is a PUBLIC API, but not annotated as such because this is not a bundle and - * we don't want to introduce Vespa dependencies. - */ -package com.yahoo.vespa.http.client.config; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java deleted file mode 100644 index 07262a60dd8..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Document.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import java.io.IOException; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.Objects; -import java.util.concurrent.ThreadLocalRandom; - -/** - * A document operation - * - * @author Einar M R Rosenvinge - */ -final public class Document { - - private final String documentId; - private final ByteBuffer data; - private final Instant createTime; - // This is initialized lazily to reduce work on calling thread (which is the thread calling the API) - private String operationId = null; - private final Object context; - private Instant queueInsertTime; - - public Document(String documentId, byte[] data, Object context, Instant createTime) { - this(documentId, null, ByteBuffer.wrap(data), context, createTime); - } - - public Document(String documentId, String operationId, CharSequence data, Object context, Instant createTime) { - this(documentId, operationId, encode(data, documentId), context, createTime); - } - - private Document(String documentId, String operationId, ByteBuffer data, Object context, Instant createTime) { - this.documentId = documentId; - this.operationId = operationId; - this.data = data; - this.context = context; - this.createTime = Objects.requireNonNull(createTime, "createTime cannot be null"); - this.queueInsertTime = createTime; - } - - public void setQueueInsertTime(Instant queueInsertTime) { - this.queueInsertTime = queueInsertTime; - } - - public Instant getQueueInsertTime() { return queueInsertTime; } - - public CharSequence getDataAsString() { - return StandardCharsets.UTF_8.decode(data.asReadOnlyBuffer()); - } - - public Object getContext() { return context; } - - public static class DocumentException extends IOException { - private static final long serialVersionUID = 29832833292L; - public DocumentException(String message) - { - super(message); - } - } - - public String getDocumentId() { return documentId; } - - public ByteBuffer getData() { - return data.asReadOnlyBuffer(); - } - - public int size() { - return data.remaining(); - } - - public Instant createTime() { return createTime; } - - public String getOperationId() { - if (operationId == null) { - operationId = new BigInteger(64, ThreadLocalRandom.current()).toString(32); - } - return operationId; - } - - @Override - public String toString() { return "document '" + documentId + "'"; } - - private static ByteBuffer encode(CharSequence data, String documentId) { - try { - return StandardCharsets.UTF_8.newEncoder().encode(CharBuffer.wrap(data)); - } catch (CharacterCodingException e) { - throw new RuntimeException("Error encoding document data into UTF8 " + documentId, e); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java deleted file mode 100644 index e4781dc3a3f..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Encoder.java +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -/** - * Simple encoding scheme to remove space, linefeed, control characters and - * anything outside ISO 646.irv:1991 from strings. The scheme is supposed to be - * human readable and debugging friendly. Opening and closing curly braces are - * used as quoting characters, the output is by definition US-ASCII only - * characters. - * - * @author Steinar Knutsen - */ -public final class Encoder { - - /** - * ISO 646.irv:1991 safe quoting into a StringBuilder instance. - * - * @param input the string to encode - * @param output the destination buffer - * @return the destination buffer given as input - */ - public static StringBuilder encode(String input, StringBuilder output) { - for (int i = 0; i < input.length(); i = input.offsetByCodePoints(i, 1)) { - int c = input.codePointAt(i); - if (c <= '~') { - if (c <= ' ') { - encode(c, output); - } else { - switch (c) { - case '{': - case '}': - encode(c, output); - break; - default: - output.append((char) c); - } - } - } else { - encode(c, output); - } - } - return output; - } - - /** - * ISO 646.irv:1991 safe unquoting into a StringBuilder instance. - * - * @param input the string to decode - * @param output the destination buffer - * @return the destination buffer given as input - * @throws IllegalArgumentException if the input string contains unexpected or invalid data - */ - public static StringBuilder decode(String input, StringBuilder output) { - for (int i = 0; i < input.length(); i = input.offsetByCodePoints(i, 1)) { - int c = input.codePointAt(i); - if (c > '~') - throw new IllegalArgumentException("Input contained character above printable ASCII at position " + i); - if (c == '{') - i = decode(input, i, output); - else - output.append((char) c); - } - return output; - } - - private static int decode(String input, int offset, StringBuilder output) { - char c = 0; - int end = offset; - int start = offset + 1; - int codePoint; - - while ('}' != c) { - if (++end >= input.length()) { - throw new IllegalArgumentException("Unterminated quoted character or empty quoting."); - } - c = input.charAt(end); - } - try { - codePoint = Integer.parseInt(input.substring(start, end), 16); - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Unexpected quoted data: [" + input.substring(start, end) + "]", e); - } - if (Character.charCount(codePoint) > 1) { - try { - output.append(Character.toChars(codePoint)); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Unexpected quoted data: [" + input.substring(start, end) + "]", e); - } - } else { - output.append((char) codePoint); - } - return end; - - } - - private static void encode(int c, StringBuilder output) { - output.append("{").append(Integer.toHexString(c)).append("}"); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java deleted file mode 100644 index 94d7237422a..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/EndpointResult.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.yahoo.vespa.http.client.Result; - -/** - * Result from a single endpoint. - * - * @author dybis - */ -public class EndpointResult { - - private final String operationId; - private final Result.Detail detail; - - public EndpointResult(String operationId, Result.Detail detail) { - this.operationId = operationId; - this.detail = detail; - } - - public String getOperationId() { - return operationId; - } - - public Result.Detail getDetail() { - return detail; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java deleted file mode 100644 index 4e739218319..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ErrorCode.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -/** - * Return types for the server. - * - * @author Einar M R Rosenvinge - * @author Steinar Knutsen - */ -public enum ErrorCode { - - OK(true, true), - ERROR(false, false), - TRANSIENT_ERROR(false, true), - END_OF_FEED(true, true); - - private boolean success; - private boolean _transient; - - ErrorCode(boolean success, boolean _transient) { - this.success = success; - this._transient = _transient; - } - - public boolean isSuccess() { - return success; - } - - public boolean isTransient() { - return _transient; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java deleted file mode 100644 index 9ff3f793756..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Exceptions.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -/** - * Helper methods for handling exceptions - * - * @author bratseth - */ -public abstract class Exceptions { - - /** - * <p>Returns a use friendly error message string which includes information from all nested exceptions.</p> - * - * <p>The form of this string is - * <code>e.getMessage(): e.getCause().getMessage(): e.getCause().getCause().getMessage()...</code> - * In addition, some heuristics are used to clean up common cases where exception nesting causes bad messages. - * </p> - */ - public static String toMessageString(Throwable t) { - StringBuilder b = new StringBuilder(); - String lastMessage = null; - String message; - for (; t != null; t = t.getCause(), lastMessage = message) { - message = getMessage(t); - if (message == null) continue; - if (message.equals(lastMessage)) continue; - if (b.length() > 0) { - b.append(": "); - } - b.append(message); - } - return b.toString(); - } - - /** Returns a useful message from *this* exception, or null if none */ - private static String getMessage(Throwable t) { - String message = t.getMessage(); - if (t.getCause() == null) { - if (message == null) return t.getClass().getSimpleName(); - } else { - if (message == null) return null; - if (message.equals(t.getCause().getClass().getName() + ": " + t.getCause().getMessage())) return null; - } - return message; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Headers.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Headers.java deleted file mode 100644 index d41f42ef652..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/Headers.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -/** - * Wrapper for shared constants used by both client and server. - * - * @author Steinar Knutsen - */ -public final class Headers { - - private Headers() { - } - - public static final String CLIENT_VERSION = "Vespa-Client-Version"; - - public static final String TIMEOUT = "X-Yahoo-Feed-Timeout"; - public static final String DRAIN = "X-Yahoo-Feed-Drain"; - public static final String ROUTE = "X-Yahoo-Feed-Route"; - public static final String VERSION = "X-Yahoo-Feed-Protocol-Version"; - public static final String SESSION_ID = "X-Yahoo-Feed-Session-Id"; - public static final String DENY_IF_BUSY = "X-Yahoo-Feed-Deny-If-Busy"; - public static final String DATA_FORMAT = "X-Yahoo-Feed-Data-Format"; - // This value can be used to route the request to a specific server when using - // several servers. It is a random value that is the same for the whole session. - public static final String SHARDING_KEY = "X-Yahoo-Feed-Sharding-Key"; - public static final String PRIORITY = "X-Yahoo-Feed-Priority"; - public static final String TRACE_LEVEL = "X-Yahoo-Feed-Trace-Level"; - - public static final int HTTP_NOT_ACCEPTABLE = 406; - - // For version 3 of the API - public static final String CLIENT_ID = "X-Yahoo-Client-Id"; - public static final String OUTSTANDING_REQUESTS = "X-Yahoo-Outstanding-Requests"; - public static final String HOSTNAME = "X-Yahoo-Hostname"; - public static final String SILENTUPGRADE = "X-Yahoo-Silent-Upgrade"; - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java deleted file mode 100644 index 34b6d8b9144..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/JsonReader.java +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonFactoryBuilder; -import com.fasterxml.jackson.core.JsonParser; -import com.yahoo.vespa.http.client.FeedClient; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Reads a stream of json documents and sends them to feedClient. - * - * @author dybis - */ -public class JsonReader { - - /** - * Max size of documents. As we stream docs in for finding doc id, we buffer the data and later stream them to - * feedclient after doc id has been revealed. - */ - private final static int maxDocumentSizeChars = 50 * 1024 * 1024; - - // Intended to be used as static. - private JsonReader() {} - - /** - * Process one inputstream and send all documents to feedclient. - * - * @param inputStream source of array of json document. - * @param feedClient where data is sent. - * @param numSent counter to be incremented for every document streamed. - */ - public static void read(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) { - try (InputStreamJsonElementBuffer jsonElementBuffer = new InputStreamJsonElementBuffer(inputStream)) { - JsonFactory jfactory = new JsonFactoryBuilder().disable(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES).build(); - JsonParser jParser = jfactory.createParser(jsonElementBuffer); - while (true) { - int documentStart = (int) jParser.getCurrentLocation().getCharOffset(); - String docId = parseOneDocument(jParser); - if (docId == null) { - int documentEnd = (int) jParser.getCurrentLocation().getCharOffset(); - int documentLength = documentEnd - documentStart; - int maxTruncatedLength = 500; - StringBuilder stringBuilder = new StringBuilder(maxTruncatedLength + 3); - for (int i = 0; i < Math.min(documentLength, maxTruncatedLength); i++) - stringBuilder.append(jsonElementBuffer.circular.get(documentStart + i)); - - if (documentLength > maxTruncatedLength) - stringBuilder.append("..."); - - throw new IllegalArgumentException("Document is missing ID: '" + stringBuilder.toString() + "'"); - } - CharSequence data = jsonElementBuffer.getJsonAsArray(jParser.getCurrentLocation().getCharOffset()); - feedClient.stream(docId, data); - numSent.incrementAndGet(); - } - } catch (EOFException ignored) { - // No more documents - } catch (IOException ioe) { - System.err.println(ioe.getMessage()); - throw new UncheckedIOException(ioe); - } - } - - /** - * This class is intended to be used with a json parser. The data is sent through this intermediate stream - * and to the parser. When the parser is done with a document, it calls postJsonAsArray which will - * stream the document up to the current position of the parser. - */ - private static class InputStreamJsonElementBuffer extends InputStreamReader { - - /** - * Simple class implementing a circular array with some custom function used for finding start and end - * of json object. The reason this is needed is that the json parser reads more than it parses - * from the input stream (seems like about 8k). Using a ByteBuffer and manually moving data - * is an order of magnitude slower than this implementation. - */ - private class CircularCharBuffer { - - int readPointer = 0; - int writePointer = 0; - final char[] data; - final int size; - - public CircularCharBuffer(int chars) { - data = new char[chars]; - size = chars; - } - - /** - * This is for throwing away [ and spaces in front of a json object, and find the position of {. - * Not for parsing much text. - * - * @return position for { - */ - public int findNextObjectStart() { - int readerPos = 0; - while (get(readerPos) != '{') { - readerPos++; - assert(readerPos<=size); - } - return readerPos; - } - - /** - * This is for throwing away comma and or ], and for finding the position of the last }. - * @param fromPos where to start searching - * @return position for } - */ - public int findLastObjectEnd(int fromPos) { - while (get(fromPos-1) != '}') { - fromPos--; - assert(fromPos >=0); - } - return fromPos; - } - - public void put(char dataByte) { - data[writePointer] = dataByte; - writePointer++; - if (writePointer >= size) writePointer = 0; - assert(writePointer != readPointer); - } - - public char get(int pos) { - int readPos = readPointer + pos; - if (readPos >= size) readPos -= size; - assert(readPos != writePointer); - return data[readPos]; - } - - public void advance(int end) { - readPointer += end; - if (readPointer >= size) readPointer -= size; - } - } - - private final CircularCharBuffer circular = new CircularCharBuffer(maxDocumentSizeChars); - private int processedChars = 0; - - public InputStreamJsonElementBuffer(InputStream inputStream) { - super(inputStream, StandardCharsets.UTF_8); - } - - /** - * Removes comma, start/end array tag (last element), spaces etc that might be surrounding a json element. - * Then sends the element to the outputstream. - * @param parserPosition how far the parser has come. Please note that the parser might have processed - * more data from the input source as it is reading chunks of data. - * @throws IOException on errors - */ - public CharSequence getJsonAsArray(long parserPosition) throws IOException { - final int charSize = (int)parserPosition - processedChars; - final int endPosOfJson = circular.findLastObjectEnd(charSize); - final int startPosOfJson = circular.findNextObjectStart(); - processedChars += charSize; - // This can be optimized since we rarely wrap the circular buffer. - StringBuilder dataBuffer = new StringBuilder(endPosOfJson - startPosOfJson); - for (int x = startPosOfJson; x < endPosOfJson; x++) { - dataBuffer.append(circular.get(x)); - } - circular.advance(charSize); - return dataBuffer.toString(); - } - - @Override - public int read(char[] b, int off, int len) throws IOException { - int length = 0; - int value = 0; - while (length < len && value != -1) { - value = read(); - if (value == -1) { - return length == 0 ? -1 : length; - } - b[off + length] = (char) value; - length++; - } - return length; - } - - @Override - public int read() throws IOException { - int value = super.read(); - if (value >= 0) circular.put((char)value); - return value; - } - } - - /** - * Parse one document from the stream and return doc id. - * - * @param jParser parser with stream. - * @return doc id of document or null if no more docs. - * @throws IOException on problems - */ - private static String parseOneDocument(JsonParser jParser) throws IOException { - int objectLevel = 0; - String documentId = null; - boolean foundObject = false; - boolean valueIsDocumentId = false; - while (jParser.nextToken() != null) { - String tokenAsText = jParser.getText(); - if (valueIsDocumentId) { - if (documentId != null) { - throw new RuntimeException("Several document ids"); - } - documentId = tokenAsText; - valueIsDocumentId = false; - } - switch(jParser.getCurrentToken()) { - case START_OBJECT: - foundObject = true; - objectLevel++; - break; - case END_OBJECT: - objectLevel--; - if (objectLevel == 0) { - return documentId; - } - break; - case FIELD_NAME: - if (objectLevel == 1 && - (tokenAsText.equals("put") - || tokenAsText.endsWith("id") - || tokenAsText.endsWith("update") - || tokenAsText.equals("remove"))) { - valueIsDocumentId = true; - } - break; - default: // No operation on all other tags. - } - } - if (!foundObject) - throw new EOFException("No more documents"); - return null; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/OperationStatus.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/OperationStatus.java deleted file mode 100644 index ee6d96aa600..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/OperationStatus.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.google.common.base.Splitter; -import java.util.Iterator; - -/** - * Serialization/deserialization class for the result of a single document operation against Vespa. - * - * @author Steinar Knutsen - */ -public final class OperationStatus { - - public static final String IS_CONDITION_NOT_MET = "IS-CONDITION-NOT-MET"; - public final String message; - public final String operationId; - public final ErrorCode errorCode; - public final String traceMessage; - public final boolean isConditionNotMet; - - private static final char EOL = '\n'; - private static final char SEPARATOR = ' '; - private static final Splitter spaceSep = Splitter.on(SEPARATOR); - - /** - * Constructor - * @param message some human readable information what happened - * @param operationId the doc ID for the operation - * @param errorCode if it is success, transitive, or fatal - * @param isConditionNotMet if error is due to condition not met - * @param traceMessage any tracemessage - */ - public OperationStatus(String message, String operationId, ErrorCode errorCode, boolean isConditionNotMet, String traceMessage) { - this.isConditionNotMet = isConditionNotMet; - this.message = message; - this.operationId = operationId; - this.errorCode = errorCode; - this.traceMessage = traceMessage; - } - - /** - * Parse a single rendered OperationStatus string. White space may be padded after - * and before the given status. - * - * @param singleLine - * a rendered OperationStatus - * @return an OperationStatus instance reflecting the input - * @throws IllegalArgumentException - * if there are illegal input data characters or the status - * element has no corresponding value in the ErrorCode - * enumeration - */ - public static OperationStatus parse(String singleLine) { - // Do note there is specifically left room for more arguments after - // the first in the serialized form. - Iterator<String> input = spaceSep.split(singleLine.trim()).iterator(); - String operationId; - ErrorCode errorCode; - String message; - String traceMessage = ""; - - operationId = Encoder.decode(input.next(), new StringBuilder()) - .toString(); - errorCode = ErrorCode.valueOf(Encoder.decode(input.next(), - new StringBuilder()).toString()); - - message = Encoder.decode(input.next(), new StringBuilder()).toString(); - // We are backwards compatible, meaning it is ok not to supply the last argument. - boolean isConditionNotMet = false; - if (message.startsWith(IS_CONDITION_NOT_MET)) { - message = message.replaceFirst(IS_CONDITION_NOT_MET, ""); - isConditionNotMet = true; - } - if (input.hasNext()) { - traceMessage = Encoder.decode(input.next(), new StringBuilder()).toString(); - } - return new OperationStatus(message, operationId, errorCode, isConditionNotMet, traceMessage); - } - - /** Returns a string representing the status. */ - public String render() { - StringBuilder s = new StringBuilder(); - Encoder.encode(operationId, s).append(SEPARATOR); - Encoder.encode(errorCode.toString(), s).append(SEPARATOR); - Encoder.encode(isConditionNotMet ? IS_CONDITION_NOT_MET + message : message, s).append(SEPARATOR); - Encoder.encode(traceMessage, s).append(EOL); - return s.toString(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java deleted file mode 100644 index d5a09d2566c..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ServerResponseException.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -/** - * The request was not processed properly on the server. - * - * @author Einar M R Rosenvinge - */ -@SuppressWarnings("serial") -public class ServerResponseException extends Exception { - - private final int responseCode; - private final String responseString; - - public ServerResponseException(int responseCode, String responseString) { - super(responseString); - this.responseCode = responseCode; - this.responseString = responseString; - } - - public ServerResponseException(String responseString) { - super(responseString); - this.responseCode = 0; - this.responseString = responseString; - } - - public int getResponseCode() { - return responseCode; - } - - public String getResponseString() { - return responseString; - } - - @Override - public String toString() { - if (responseCode > 0) { - return responseCode + ": " + responseString; - } - return responseString; - } - -} - diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java deleted file mode 100644 index 101bd001fb8..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/ThrottlePolicy.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import static java.lang.Math.abs; -import static java.lang.Math.max; -import static java.lang.Math.min; - -/** - * Class that has a method for finding next maxInFlight. - * - * @author dybis - */ -public class ThrottlePolicy { - - public static final double SMALL_DIFFERENCE_IN_SUCCESSES_RATIO = 0.15; - private static final double MINIMUM_DIFFERENCE = 0.05; - - /** - * Generate nex in-flight value for throttling. - * - * @param maxPerformanceChange This value limit the dynamics of the algorithm. - * @param numOk number of success in last phase - * @param previousNumOk number of success in previous (before last) phase. - * @param previousMaxInFlight number of max-in-flight in previous (before last) phase. - * @param maxInFlightNow number of max-in-flight in last phase. - * @param messagesQueued if any messages where queued. - * @return The new value to be used for max-in-flight (should be cropped externally to fit max/min values). - */ - public int calcNewMaxInFlight(double maxPerformanceChange, int numOk, int previousNumOk, int previousMaxInFlight, - int maxInFlightNow, boolean messagesQueued) { - - double difference = calculateRuleBasedDifference(maxPerformanceChange, numOk, previousNumOk, previousMaxInFlight, maxInFlightNow); - boolean previousRunWasBetter = numOk < previousNumOk; - boolean previousRunHadLessInFlight = previousMaxInFlight < maxInFlightNow; - - - int delta; - if (previousRunWasBetter == previousRunHadLessInFlight) { - delta = (int) (-1.1 * difference * maxInFlightNow); - } else { - delta = (int) (difference * maxInFlightNow); - } - - // We don't want the same size since we need different sizes for algorithm to adjust. - if (abs(delta) < 2) { - delta = -3; - } - // We never used all permits in previous run, no reason to grow more, we should rather reduce permits. - if (!messagesQueued && delta > 0) { - delta = -2; - } - return maxInFlightNow + delta; - } - - private static double calculateRuleBasedDifference(double maxPerformanceChange, double numOk, double previousNumOk, - double previousMaxInFlight, double maxInFlightNow) { - double difference = min( - maxPerformanceChange, - abs((numOk - previousNumOk) / safeDenominator(previousNumOk))); - - if (abs(previousMaxInFlight - maxInFlightNow) / safeDenominator(min(previousMaxInFlight, maxInFlightNow)) - < SMALL_DIFFERENCE_IN_SUCCESSES_RATIO) { - difference = min(difference, 0.2); - } - - // We want some changes so we can track performance as a result of different throttling. - return max(difference, MINIMUM_DIFFERENCE); - } - - private static double safeDenominator(double x) { - return x == 0.0 ? 1.0 : x; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java deleted file mode 100644 index 349959f496e..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/XmlFeedReader.java +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.yahoo.vespa.http.client.FeedClient; -import org.xml.sax.Attributes; -import org.xml.sax.InputSource; -import org.xml.sax.ext.DefaultHandler2; - -import javax.xml.parsers.SAXParser; -import javax.xml.parsers.SAXParserFactory; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Reads an input stream of xml, sends these to session. - * - * @author dybis -*/ -public class XmlFeedReader { - - // Static class. - private XmlFeedReader() {} - - public static void read(InputStream inputStream, FeedClient feedClient, AtomicInteger numSent) throws Exception { - SAXParserFactory parserFactory = SAXParserFactory.newInstance(); - // XXE prevention: - parserFactory.setFeature("http://xml.org/sax/features/external-general-entities", false); - parserFactory.setValidating(false); - parserFactory.setNamespaceAware(false); - SAXParser parser = parserFactory.newSAXParser(); - SAXClientFeeder saxClientFeeder = new SAXClientFeeder(feedClient, numSent); - - InputSource inputSource = new InputSource(); - inputSource.setEncoding(StandardCharsets.UTF_8.displayName()); - inputSource.setByteStream(inputStream); - // This is to send events about CDATA to the saxClientFeeder - // (https://docs.oracle.com/javase/tutorial/jaxp/sax/events.html) - parser.setProperty("http://xml.org/sax/properties/lexical-handler", saxClientFeeder); - parser.parse(inputSource, saxClientFeeder); - } -} - -/** - * Streams XML and sends each document operation to feeder. - */ -class SAXClientFeeder extends DefaultHandler2 { - - public static final String CDATA_START = "<![CDATA["; - public static final String CDATA_STOP = "]]>"; - private final FeedClient feedClient; - int vespaIndent = 0; - int documentIndent = 0; - String documentId = null; - StringBuilder content = new StringBuilder(); - final AtomicInteger numSent; - boolean isCData = false; - - public SAXClientFeeder(FeedClient feedClient, AtomicInteger numSent) { - this.feedClient = feedClient; - this.numSent = numSent; - } - - @Override - public void startCDATA() { - content.append(CDATA_START); - isCData = true; - } - - @Override - public void endCDATA() { - content.append(CDATA_STOP); - isCData = false; - } - - @Override - public void comment(char[] ch, int start, int length) { } - - @SuppressWarnings("fallthrough") - @Override - public void startElement(String uri, String localName, String qName, Attributes attributes) { - switch(qName){ - case "vespafeed": - vespaIndent++; - if (vespaIndent == 1 && documentIndent == 0) { - // If this is the first vespafeed tag, it should not be added to content of the first item. - return; - } - case "update": - case "remove": - case "document" : - documentIndent++; - documentId = attributes.getValue("documentid"); - content = new StringBuilder(); - } - content.append("<" + qName); - if (attributes != null) { - for (int i = 0; i < attributes.getLength (); i++) { - content.append(" ") - .append(attributes.getQName(i)) - .append("=\""); - String attributesValue = attributes.getValue(i); - characters(attributesValue.toCharArray(), 0, attributesValue.length()); - content.append("\""); - } - } - content.append(">"); - } - - @Override - public void endElement(String uri, String localName, String qName) { - content.append("</") - .append(qName) - .append(">"); - switch(qName){ - case "vespafeed": - vespaIndent--; - return; - case "update": - case "remove": - case "document" : - documentIndent--; - if (documentIndent == 0) { - if (documentId == null || documentId.isEmpty()) { - throw new IllegalArgumentException("no docid"); - } - feedClient.stream(documentId, content); - numSent.incrementAndGet(); - } - } - } - - @Override - public void characters (char buf [], int offset, int len) { - if (isCData) { - content.append(buf, offset, len); - return; - } - - // This is on the critical loop for performance, otherwise a library would have been used. - // We can do a few shortcuts as well as this data is already decoded by SAX parser. - for (int x = offset ; x < len + offset ; x++) { - switch (buf[x]) { - case '&' : content.append("&"); continue; - case '<' : content.append("<"); continue; - case '>' : content.append(">"); continue; - case '"' : content.append("""); continue; - case '\'' : content.append("'"); continue; - default: content.append(buf[x]); continue; - } - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java deleted file mode 100644 index b46f23923f5..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/FeedClientImpl.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.api; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.ThrottlePolicy; -import com.yahoo.vespa.http.client.core.operationProcessor.IncompleteResultsThrottler; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.nio.charset.CharsetEncoder; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.time.Instant; -import java.util.Optional; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Logger; - -/** - * Implementation of FeedClient. It is a thin layer on top of multiClusterHandler and multiClusterResultAggregator. - * - * @author dybis - */ -public class FeedClientImpl implements FeedClient { - - private static final Logger log = Logger.getLogger(FeedClientImpl.class.getName()); - private static final AtomicBoolean warningPrinted = new AtomicBoolean(false); - - private final Clock clock; - private final OperationProcessor operationProcessor; - private final long closeTimeoutMs; - private final long sleepTimeMs = 500; - - public FeedClientImpl(SessionParams sessionParams, - ResultCallback resultCallback, - ScheduledThreadPoolExecutor timeoutExecutor, - Clock clock) { - this.clock = clock; - this.closeTimeoutMs = (10 + 3 * sessionParams.getConnectionParams().getMaxRetries()) * - (sessionParams.getFeedParams().getServerTimeout(TimeUnit.MILLISECONDS) + - sessionParams.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS)); - this.operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(sessionParams.getThrottlerMinSize(), - sessionParams.getClientQueueSize(), - clock, - new ThrottlePolicy()), - resultCallback, - sessionParams, - timeoutExecutor, - clock); - if (warningPrinted.compareAndSet(false, true)) { - log.warning("The vespa-http-client is deprecated and will be removed on Vespa 8. " + - "See https://docs.vespa.ai/en/vespa8-release-notes.html"); - } - } - - @Override - public void stream(String documentId, String operationId, CharSequence documentData, Object context) { - CharsetEncoder charsetEncoder = StandardCharsets.UTF_8.newEncoder(); - charsetEncoder.onMalformedInput(CodingErrorAction.REPORT); - charsetEncoder.onUnmappableCharacter(CodingErrorAction.REPORT); - - Document document = new Document(documentId, operationId, documentData, context, clock.instant()); - operationProcessor.sendDocument(document); - } - - @Override - public void close() { - Instant lastOldestResultReceivedAt = Instant.now(); - Optional<String> oldestIncompleteId = operationProcessor.oldestIncompleteResultId(); - - while (oldestIncompleteId.isPresent() && waitForOperations(lastOldestResultReceivedAt, sleepTimeMs, closeTimeoutMs)) { - Optional<String> oldestIncompleteIdNow = operationProcessor.oldestIncompleteResultId(); - if ( ! oldestIncompleteId.equals(oldestIncompleteIdNow)) - lastOldestResultReceivedAt = Instant.now(); - oldestIncompleteId = oldestIncompleteIdNow; - } - operationProcessor.close(); - } - - @Override - public String getStatsAsJson() { - return operationProcessor.getStatsAsJson(); - } - - // On return value true, wait more. Public for testing. - public static boolean waitForOperations(Instant lastResultReceived, long sleepTimeMs, long closeTimeoutMs) { - if (lastResultReceived.plusMillis(closeTimeoutMs).isBefore(Instant.now())) { - return false; - } - try { - Thread.sleep(sleepTimeMs); - } catch (InterruptedException e) { - return false; - } - return true; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java deleted file mode 100644 index 27fc22b9675..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/MultiClusterSessionOutputStream.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.api; - -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.time.Clock; - -/** - * Class for wiring up the Session API. It is the return value of stream() in the Session API. - * - * @author dybis -*/ -class MultiClusterSessionOutputStream extends ByteArrayOutputStream { - - private final CharSequence documentId; - private final OperationProcessor operationProcessor; - private final Object context; - private final Clock clock; - - public MultiClusterSessionOutputStream(CharSequence documentId, - OperationProcessor operationProcessor, - Object context, - Clock clock) { - this.documentId = documentId; - this.context = context; - this.operationProcessor = operationProcessor; - this.clock = clock; - } - - @Override - public void close() throws IOException { - Document document = new Document(documentId.toString(), toByteArray(), context, clock.instant()); - operationProcessor.sendDocument(document); - super.close(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java deleted file mode 100644 index 05b66ac4a46..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.api; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.ThrottlePolicy; -import com.yahoo.vespa.http.client.core.operationProcessor.IncompleteResultsThrottler; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.io.OutputStream; -import java.time.Clock; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledThreadPoolExecutor; - -/** - * This class wires up the Session API using MultiClusterHandler and MultiClusterSessionOutputStream. - * - * @deprecated - */ -@Deprecated // TODO: Remove on Vespa 8 -public class SessionImpl implements com.yahoo.vespa.http.client.Session { - - private final OperationProcessor operationProcessor; - private final BlockingQueue<Result> resultQueue = new LinkedBlockingQueue<>(); - private final Clock clock; - - public SessionImpl(SessionParams sessionParams, ScheduledThreadPoolExecutor timeoutExecutor, Clock clock) { - this.clock = clock; - this.operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler( - sessionParams.getThrottlerMinSize(), - sessionParams.getClientQueueSize(), - clock, - new ThrottlePolicy()), - new FeedClient.ResultCallback() { - @Override - public void onCompletion(String docId, Result documentResult) { - resultQueue.offer(documentResult); - } - }, - sessionParams, - timeoutExecutor, - clock); - } - - @Override - public OutputStream stream(CharSequence documentId) { - return new MultiClusterSessionOutputStream(documentId, operationProcessor, null, clock); - } - - @Override - public BlockingQueue<Result> results() { - return resultQueue; - } - - @Override - public void close() { - operationProcessor.close(); - } - - @Override - public String getStatsAsJson() { - return operationProcessor.getStatsAsJson(); - } - - // For testing only (legacy tests). - public int getIncompleteResultQueueSize() { - return operationProcessor.getIncompleteResultQueueSize(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java deleted file mode 100644 index 6b1078fa393..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ /dev/null @@ -1,507 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import ai.vespa.util.http.hc4.VespaHttpClientBuilder; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.security.SslContextBuilder; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.Encoder; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import com.yahoo.vespa.http.client.core.Vtag; -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequest; -import org.apache.http.HttpResponse; -import org.apache.http.StatusLine; -import org.apache.http.auth.AUTH; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.ChallengeState; -import org.apache.http.auth.Credentials; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.auth.BasicScheme; -import org.apache.http.impl.client.BasicAuthCache; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.message.BasicHeader; -import org.apache.http.protocol.BasicHttpContext; -import org.apache.http.protocol.HttpContext; -import org.apache.http.util.EntityUtils; - -import javax.net.ssl.SSLContext; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.zip.GZIPOutputStream; - -/** - * @author Einar M R Rosenvinge - */ -class ApacheGatewayConnection implements GatewayConnection { - - private static final Logger log = Logger.getLogger(ApacheGatewayConnection.class.getName()); - private static final ObjectMapper mapper = new ObjectMapper(); - private static final String PATH = "/reserved-for-internal-use/feedapi?"; - private static final byte[] START_OF_FEED_XML = "<vespafeed>\n".getBytes(StandardCharsets.UTF_8); - private static final byte[] END_OF_FEED_XML = "\n</vespafeed>\n".getBytes(StandardCharsets.UTF_8); - private static final byte[] START_OF_FEED_JSON = "[".getBytes(StandardCharsets.UTF_8); - private static final byte[] END_OF_FEED_JSON = "]".getBytes(StandardCharsets.UTF_8); - - private final List<Integer> supportedVersions = new ArrayList<>(); - private final byte[] startOfFeed; - private final byte[] endOfFeed; - private final Endpoint endpoint; - private final FeedParams feedParams; - private final String clusterSpecificRoute; - private final ConnectionParams connectionParams; - private CloseableHttpClient httpClient; - private Instant connectionTime = null; - private Instant lastPollTime = null; - private String sessionId; - private final String clientId; - private int negotiatedVersion = -1; - private final HttpClientFactory httpClientFactory; - private final String shardingKey = UUID.randomUUID().toString().substring(0, 5); - private final Clock clock; - - ApacheGatewayConnection(Endpoint endpoint, - FeedParams feedParams, - String clusterSpecificRoute, - ConnectionParams connectionParams, - HttpClientFactory httpClientFactory, - String clientId, - Clock clock) { - supportedVersions.add(3); - this.endpoint = endpoint; - this.feedParams = feedParams; - this.clusterSpecificRoute = clusterSpecificRoute; - this.httpClientFactory = httpClientFactory; - this.connectionParams = connectionParams; - this.httpClient = null; - this.clientId = clientId; - this.clock = clock; - - if (feedParams.getDataFormat() == FeedParams.DataFormat.JSON_UTF8) { - startOfFeed = START_OF_FEED_JSON; - endOfFeed = END_OF_FEED_JSON; - } else { - startOfFeed = START_OF_FEED_XML; - endOfFeed = END_OF_FEED_XML; - } - } - - @Override - public InputStream write(List<Document> docs) throws ServerResponseException, IOException { - return write(docs, false, connectionParams.getUseCompression()); - } - - @Override - public InputStream poll() throws ServerResponseException, IOException { - lastPollTime = clock.instant(); - return write(Collections.<Document>emptyList(), false, false); - } - - @Override - public Instant lastPollTime() { return lastPollTime; } - - @Override - public InputStream drain() throws ServerResponseException, IOException { - return write(Collections.<Document>emptyList(), true, false); - } - - @Override - public boolean connect() { - log.fine(() -> "Attempting to connect to " + endpoint); - if (httpClient != null) - log.log(Level.WARNING, "Previous httpClient still exists."); - httpClient = httpClientFactory.createClient(); - connectionTime = clock.instant(); - return httpClient != null; - } - - @Override - public Instant connectionTime() { return connectionTime; } - - // Protected for easier testing only. - protected static InputStreamEntity zipAndCreateEntity(final InputStream inputStream) throws IOException { - byte[] buffer = new byte[4096]; - GZIPOutputStream gzos = null; - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try { - gzos = new GZIPOutputStream(baos); - while (inputStream.available() > 0) { - int length = inputStream.read(buffer); - gzos.write(buffer, 0,length); - } - } finally { - if (gzos != null) { - gzos.close(); - } - } - byte[] fooGzippedBytes = baos.toByteArray(); - return new InputStreamEntity(new ByteArrayInputStream(fooGzippedBytes), -1); - } - - private InputStream write(List<Document> docs, boolean drain, boolean useCompression) - throws ServerResponseException, IOException { - HttpPost httpPost = createPost(drain, useCompression, false); - - ByteBuffer[] buffers = getDataWithStartAndEndOfFeed(docs, negotiatedVersion); - InputStream inputStream = new ByteBufferInputStream(buffers); - InputStreamEntity reqEntity = useCompression ? zipAndCreateEntity(inputStream) - : new InputStreamEntity(inputStream, -1); - reqEntity.setChunked(true); - httpPost.setEntity(reqEntity); - return executePost(httpPost); - } - - private ByteBuffer[] getDataWithStartAndEndOfFeed(List<Document> docs, int version) { - List<ByteBuffer> data = new ArrayList<>(); - if (version == 3) { - for (Document doc : docs) { - int operationSize = doc.size() + startOfFeed.length + endOfFeed.length; - StringBuilder envelope = new StringBuilder(); - Encoder.encode(doc.getOperationId(), envelope); - envelope.append(' '); - envelope.append(Integer.toHexString(operationSize)); - envelope.append('\n'); - data.add(StandardCharsets.US_ASCII.encode(envelope.toString())); - data.add(ByteBuffer.wrap(startOfFeed)); - data.add(doc.getData()); - data.add(ByteBuffer.wrap(endOfFeed)); - } - } else { - throw new IllegalArgumentException("Protocol version " + version + " unsupported by client."); - } - return data.toArray(new ByteBuffer[data.size()]); - } - - private HttpPost createPost(boolean drain, boolean useCompression, boolean isHandshake) { - HttpPost httpPost = new HttpPost(createUri()); - - for (int v : supportedVersions) { - httpPost.addHeader(Headers.VERSION, "" + v); - } - if (sessionId != null) { - httpPost.setHeader(Headers.SESSION_ID, sessionId); - } - if (clientId != null) { - httpPost.setHeader(Headers.CLIENT_ID, clientId); - } - httpPost.setHeader(Headers.SHARDING_KEY, shardingKey); - httpPost.setHeader(Headers.DRAIN, drain ? "true" : "false"); - if (clusterSpecificRoute != null) { - httpPost.setHeader(Headers.ROUTE, feedParams.getRoute()); - } else { - if (feedParams.getRoute() != null) { - httpPost.setHeader(Headers.ROUTE, feedParams.getRoute()); - } - } - if (!isHandshake) { - if (feedParams.getDataFormat() == FeedParams.DataFormat.JSON_UTF8) { - httpPost.setHeader(Headers.DATA_FORMAT, FeedParams.DataFormat.JSON_UTF8.name()); - } else { - httpPost.setHeader(Headers.DATA_FORMAT, FeedParams.DataFormat.XML_UTF8.name()); - } - if (feedParams.getPriority() != null) { - httpPost.setHeader(Headers.PRIORITY, feedParams.getPriority()); - } - if (connectionParams.getTraceLevel() != 0) { - httpPost.setHeader(Headers.TRACE_LEVEL, String.valueOf(connectionParams.getTraceLevel())); - } - if (negotiatedVersion == 3 && feedParams.getDenyIfBusyV3()) { - httpPost.setHeader(Headers.DENY_IF_BUSY, "true"); - } - } - if (feedParams.getSilentUpgrade()) { - httpPost.setHeader(Headers.SILENTUPGRADE, "true"); - } - httpPost.setHeader(Headers.TIMEOUT, "" + feedParams.getServerTimeout(TimeUnit.SECONDS)); - - for (Map.Entry<String, String> extraHeader : connectionParams.getHeaders()) { - httpPost.addHeader(extraHeader.getKey(), extraHeader.getValue()); - } - connectionParams.getDynamicHeaders().forEach((headerName, provider) -> { - String headerValue = Objects.requireNonNull( - provider.getHeaderValue(), - provider.getClass().getName() + ".getHeader() returned null as header value!"); - httpPost.addHeader(headerName, headerValue); - }); - - if (useCompression) { // This causes the apache client to gzip the request content. Weird, huh? - httpPost.setHeader("Content-Encoding", "gzip"); - } - return httpPost; - } - - private InputStream executePost(HttpPost httpPost) throws ServerResponseException, IOException { - if (httpClient == null) - throw new IOException("Trying to executePost while not having a connection/http client"); - String proxyAuthzHeader = getCustomProxyAuthorizationHeader(connectionParams).orElse(null); - HttpResponse response; - if (connectionParams.getProxyHost() != null && proxyAuthzHeader != null) { - HttpContext context = createContextForcingPreemptiveProxyAuth(proxyAuthzHeader); - response = httpClient.execute(httpPost, context); - } else { - response = httpClient.execute(httpPost); - } - try { - verifyServerResponseCode(response); - verifyServerVersion(response.getFirstHeader(Headers.VERSION)); - verifySessionHeader(response.getFirstHeader(Headers.SESSION_ID)); - } catch (ServerResponseException e) { - // Ensure response is consumed to allow connection reuse later on - EntityUtils.consumeQuietly(response.getEntity()); - throw e; - } - // Consume response now to allow connection to be reused immediately - byte[] responseData = EntityUtils.toByteArray(response.getEntity()); - return responseData == null ? null : new ByteArrayInputStream(responseData); - } - - private static Optional<String> getCustomProxyAuthorizationHeader(ConnectionParams params) { - return params.getHeaders().stream() - .filter(h -> h.getKey().equals(AUTH.PROXY_AUTH_RESP)) - .findAny() - .map(Map.Entry::getValue); - } - - private HttpContext createContextForcingPreemptiveProxyAuth(String proxyAuthzHeader) { - BasicAuthCache authCache = new BasicAuthCache(); - HttpHost proxy = new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort()); - authCache.put(proxy, new CustomAuthScheme(proxyAuthzHeader)); - HttpContext context = new BasicHttpContext(); - context.setAttribute(HttpClientContext.AUTH_CACHE, authCache); - BasicCredentialsProvider prov = new BasicCredentialsProvider(); - prov.setCredentials(new AuthScope(proxy), new UsernamePasswordCredentials("", "")); - context.setAttribute(HttpClientContext.CREDS_PROVIDER, prov); - return context; - } - private static class CustomAuthScheme extends BasicScheme { - final String proxyAuthzHeader; - @SuppressWarnings("deprecation") - CustomAuthScheme(String proxyAuthzHeader) { - super(ChallengeState.PROXY); - this.proxyAuthzHeader = proxyAuthzHeader; - } - @Override - public Header authenticate(Credentials credentials, HttpRequest request, HttpContext context) { - return new BasicHeader(AUTH.PROXY_AUTH_RESP, proxyAuthzHeader); - } - } - - - private void verifyServerResponseCode(HttpResponse response) throws ServerResponseException { - StatusLine statusLine = response.getStatusLine(); - int statusCode = statusLine.getStatusCode(); - - // We use code 261-299 to report errors related to internal transitive errors that the tenants should not care - // about to avoid masking more serious errors. - if (statusCode > 199 && statusCode < 260) return; - if (statusCode == 299) throw new ServerResponseException(429, "Too many requests."); - throw new ServerResponseException(statusCode, - tryGetDetailedErrorMessage(response).orElseGet(statusLine::getReasonPhrase)); - } - - private static Optional<String> tryGetDetailedErrorMessage(HttpResponse response) { - Header contentType = response.getEntity().getContentType(); - if (contentType == null || !contentType.getValue().equalsIgnoreCase("application/json")) return Optional.empty(); - try (InputStream in = response.getEntity().getContent()) { - JsonNode jsonNode = mapper.readTree(in); - JsonNode message = jsonNode.get("message"); - if (message == null || message.textValue() == null) return Optional.empty(); - return Optional.of(response.getStatusLine().getReasonPhrase() + " - " + message.textValue()); - } catch (IOException e) { - return Optional.empty(); - } - } - - private void verifySessionHeader(Header serverHeader) throws ServerResponseException { - if (serverHeader == null) { - throw new ServerResponseException("Got no session ID from server."); - } - final String serverHeaderVal = serverHeader.getValue().trim(); - if (negotiatedVersion == 3) { - if (clientId == null || !clientId.equals(serverHeaderVal)) { - String message = "Running using v3. However, server responds with different session " + - "than client has set; " + serverHeaderVal + " vs client code " + clientId; - log.severe(message); - throw new ServerResponseException(message); - } - return; - } - if (sessionId == null) { //this must be the first request - log.finer("Got session ID from server: " + serverHeaderVal); - this.sessionId = serverHeaderVal; - } else { - if (!sessionId.equals(serverHeaderVal)) { - log.info("Request has been routed to a server which does not recognize the client session." + - " Most likely cause is upgrading of cluster, transitive error."); - throw new ServerResponseException("Session ID received from server ('" + serverHeaderVal + - "') does not match cached session ID ('" + sessionId + "')"); - } - } - } - - private void verifyServerVersion(Header serverHeader) throws ServerResponseException { - if (serverHeader == null) { - throw new ServerResponseException("Got bad protocol version from server."); - } - int serverVersion; - try { - serverVersion = Integer.parseInt(serverHeader.getValue()); - } catch (NumberFormatException nfe) { - throw new ServerResponseException("Got bad protocol version from server: " + nfe.getMessage()); - } - if (!supportedVersions.contains(serverVersion)) { - throw new ServerResponseException("Unsupported version: " + serverVersion - + ". Supported versions: " + supportedVersions); - } - if (negotiatedVersion == -1) { - if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, "Server decided upon protocol version " + serverVersion + "."); - } - } - this.negotiatedVersion = serverVersion; - } - - private String createUri() { - StringBuilder u = new StringBuilder(); - u.append(endpoint.isUseSsl() ? "https://" : "http://"); - u.append(endpoint.getHostname()); - u.append(":").append(endpoint.getPort()); - u.append(PATH); - u.append(feedParams.toUriParameters()); - return u.toString(); - } - - @Override - public Endpoint getEndpoint() { - return endpoint; - } - - @Override - public void handshake() throws ServerResponseException, IOException { - boolean useCompression = false; - boolean drain = false; - boolean handshake = true; - HttpPost httpPost = createPost(drain, useCompression, handshake); - - String oldSessionID = sessionId; - sessionId = null; - try (InputStream stream = executePost(httpPost)) { - if (oldSessionID != null && !oldSessionID.equals(sessionId)) { - throw new ServerResponseException( - "Session ID changed after new handshake, some documents might not be acked to correct thread. " - + getEndpoint() + " old " + oldSessionID + " new " + sessionId); - } - if (stream == null) { - log.fine("Stream is null."); - } - log.fine("Got session ID " + sessionId); - } - } - - @Override - public void close() { - try { - if (httpClient != null) - httpClient.close(); - } - catch (IOException e) { - log.log(Level.WARNING, "Failed closing HTTP client", e); - } - httpClient = null; - } - - /** - * On re-connect we want to recreate the connection, hence we need a factory. - */ - public static class HttpClientFactory { - - private final FeedParams feedParams; - final ConnectionParams connectionParams; - final boolean useSsl; - - public HttpClientFactory(FeedParams feedParams, ConnectionParams connectionParams, boolean useSsl) { - this.feedParams = feedParams; - this.connectionParams = connectionParams; - this.useSsl = useSsl; - } - - public CloseableHttpClient createClient() { - HttpClientBuilder clientBuilder; - if (connectionParams.useTlsConfigFromEnvironment()) { - clientBuilder = VespaHttpClientBuilder.create(); - } else { - clientBuilder = HttpClientBuilder.create(); - if (connectionParams.getSslContext() != null) { - setSslContext(clientBuilder, connectionParams.getSslContext()); - } else { - SslContextBuilder builder = new SslContextBuilder(); - if (connectionParams.getPrivateKey() != null && connectionParams.getCertificate() != null) { - builder.withKeyStore(connectionParams.getPrivateKey(), connectionParams.getCertificate()); - } - if (connectionParams.getCaCertificates() != null) { - builder.withTrustStore(connectionParams.getCaCertificates()); - } - setSslContext(clientBuilder, builder.build()); - } - if (connectionParams.getHostnameVerifier() != null) { - clientBuilder.setSSLHostnameVerifier(connectionParams.getHostnameVerifier()); - } - clientBuilder.setUserTokenHandler(context -> null); // https://stackoverflow.com/a/42112034/1615280 - } - clientBuilder.setMaxConnPerRoute(1); - clientBuilder.setMaxConnTotal(1); - clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.V_TAG_COMPONENT)); - clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.V_TAG_COMPONENT))); - int millisTotalTimeout = (int) (feedParams.getClientTimeout(TimeUnit.MILLISECONDS) + feedParams.getServerTimeout(TimeUnit.MILLISECONDS)); - RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() - .setSocketTimeout(millisTotalTimeout) - .setConnectTimeout(millisTotalTimeout); - if (connectionParams.getProxyHost() != null) { - requestConfigBuilder.setProxy(new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort())); - } - clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); - - log.fine(() -> "Creating HttpClient:" + - " ConnectionTimeout " + connectionParams.getConnectionTimeToLive().getSeconds() + " seconds" + - " proxyhost (can be null) " + connectionParams.getProxyHost() + ":" + connectionParams.getProxyPort() - + (useSsl ? " using ssl " : " not using ssl") - ); - return clientBuilder.build(); - } - } - - // Note: Using deprecated setSslContext() to allow httpclient 4.4 on classpath (e.g unexpected Maven dependency resolution for test classpath) - @SuppressWarnings("deprecation") - private static void setSslContext(HttpClientBuilder builder, SSLContext sslContext) { - builder.setSslcontext(sslContext); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionFactory.java deleted file mode 100644 index 0b02626d6e8..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionFactory.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.time.Clock; -import java.util.Objects; - -/** - * @author bratseth - */ -public class ApacheGatewayConnectionFactory implements GatewayConnectionFactory { - - private final Endpoint endpoint; - private final FeedParams feedParams; - private final String clusterSpecificRoute; - private final ConnectionParams connectionParams; - private final ApacheGatewayConnection.HttpClientFactory httpClientFactory; - private final String clientId; - private final Clock clock; - - public ApacheGatewayConnectionFactory(Endpoint endpoint, - FeedParams feedParams, - String clusterSpecificRoute, - ConnectionParams connectionParams, - ApacheGatewayConnection.HttpClientFactory httpClientFactory, - String clientId, - Clock clock) { - this.endpoint = validate(endpoint); - this.feedParams = feedParams; - this.clusterSpecificRoute = clusterSpecificRoute; - this.httpClientFactory = httpClientFactory; - this.connectionParams = connectionParams; - this.clientId = Objects.requireNonNull(clientId, "clientId cannot be null"); - this.clock = clock; - } - - private static Endpoint validate(Endpoint endpoint) { - try { - InetAddress.getByName(endpoint.getHostname()); - return endpoint; - } - catch (UnknownHostException e) { - throw new IllegalArgumentException("Unknown host: " + endpoint); - } - } - - @Override - public GatewayConnection newConnection() { - return new ApacheGatewayConnection(endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - httpClientFactory, - clientId, - clock); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java deleted file mode 100644 index f88519c2615..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStream.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.Deque; - -/** - * @author Einar M R Rosenvinge - */ -class ByteBufferInputStream extends InputStream { - private final Deque<ByteBuffer> currentBuffers = new ArrayDeque<>(); - - ByteBufferInputStream(ByteBuffer[] buffers) { - for (int i = buffers.length - 1; i > -1; i--) { - currentBuffers.push(buffers[i]); - } - } - - @Override - public int read() throws IOException { - pop(); - if (currentBuffers.isEmpty()) { - return -1; - } - return currentBuffers.peek().get(); - } - - private void pop() { - if (currentBuffers.isEmpty()) { - return; - } - - while (!currentBuffers.isEmpty() && !currentBuffers.peek().hasRemaining()) { - //it's exhausted, get rid of it - currentBuffers.pop(); - } - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - if (b == null) { - throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { - throw new IndexOutOfBoundsException(); - } else if (len == 0) { - return 0; - } - pop(); - if (currentBuffers.isEmpty()) { - return -1; - } - int toRead = Math.min(len, currentBuffers.peek().remaining()); - currentBuffers.peek().get(b, off, toRead); - return toRead; - } - - @Override - public long skip(long n) throws IOException { - throw new IOException("skip() not supported."); - } - - @Override - public int available() throws IOException { - if (currentBuffers.isEmpty()) { - return 0; - } - - int size = 0; - for (ByteBuffer b : currentBuffers) { - size += b.remaining(); - } - return size; - } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java deleted file mode 100644 index ef5b7afd16a..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ClusterConnection.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.Exceptions; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.io.IOException; -import java.io.StringWriter; -import java.time.Clock; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - * @author Einar M R Rosenvinge - */ -public class ClusterConnection implements AutoCloseable { - - private static ObjectMapper jsonMapper = createMapper(); - - private final List<IOThread> ioThreads = new ArrayList<>(); - private final int clusterId; - private final ThreadGroup ioThreadGroup; - - /** The shared queue of document operations the io threads will take from */ - private final DocumentQueue documentQueue; - - /** The single endpoint this sends to, or null if it will send to multiple endpoints */ - private final Endpoint singleEndpoint; - - public ClusterConnection(OperationProcessor operationProcessor, - FeedParams feedParams, - ConnectionParams connectionParams, - Cluster cluster, - int clusterId, - int clientQueueSizePerCluster, - ScheduledThreadPoolExecutor timeoutExecutor, - Clock clock) { - if (cluster.getEndpoints().isEmpty()) - throw new IllegalArgumentException("At least a single endpoint is required in " + cluster); - - this.clusterId = clusterId; - int totalNumberOfEndpointsInThisCluster = cluster.getEndpoints().size() * connectionParams.getNumPersistentConnectionsPerEndpoint(); - if (totalNumberOfEndpointsInThisCluster == 0) - throw new IllegalArgumentException("At least 1 persistent connection per endpoint is required in " + cluster); - int maxInFlightPerSession = Math.max(1, feedParams.getMaxInFlightRequests() / totalNumberOfEndpointsInThisCluster); - - documentQueue = new DocumentQueue(clientQueueSizePerCluster, clock); - ioThreadGroup = operationProcessor.getIoThreadGroup(); - singleEndpoint = cluster.getEndpoints().size() == 1 ? cluster.getEndpoints().get(0) : null; - Double idlePollFrequency = feedParams.getIdlePollFrequency(); - if (idlePollFrequency == null) - idlePollFrequency = 10.0; - for (Endpoint endpoint : cluster.getEndpoints()) { - EndpointResultQueue endpointResultQueue = new EndpointResultQueue(operationProcessor, - endpoint, - clusterId, - timeoutExecutor, - feedParams.getServerTimeout(TimeUnit.MILLISECONDS) + feedParams.getClientTimeout(TimeUnit.MILLISECONDS)); - for (int i = 0; i < connectionParams.getNumPersistentConnectionsPerEndpoint(); i++) { - GatewayConnectionFactory connectionFactory; - if (connectionParams.isDryRun()) { - connectionFactory = new DryRunGatewayConnectionFactory(endpoint, clock); - } else { - connectionFactory = new ApacheGatewayConnectionFactory(endpoint, - feedParams, - cluster.getRoute(), - connectionParams, - new ApacheGatewayConnection.HttpClientFactory(feedParams, connectionParams, endpoint.isUseSsl()), - operationProcessor.getClientId(), - clock - ); - } - IOThread ioThread = new IOThread(operationProcessor.getIoThreadGroup(), - endpoint, - endpointResultQueue, - connectionFactory, - clusterId, - feedParams.getMaxChunkSizeBytes(), - maxInFlightPerSession, - Duration.ofMillis(feedParams.getLocalQueueTimeOut()), - documentQueue, - feedParams.getMaxSleepTimeMs(), - connectionParams.getConnectionTimeToLive(), - connectionParams.runThreads(), - idlePollFrequency, - clock); - ioThreads.add(ioThread); - } - } - } - - private static ObjectMapper createMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new Jdk8Module()); - mapper.registerModule(new JavaTimeModule()); - return mapper; - } - - public int getClusterId() { - return clusterId; - } - - public void post(Document document) throws EndpointIOException { - try { - documentQueue.put(document, Thread.currentThread().getThreadGroup() == ioThreadGroup); - } catch (Throwable t) { // InterruptedException if shutting down, IllegalStateException if already shut down - throw new EndpointIOException(singleEndpoint, "While sending", t); - } - } - - @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - @Override - public void close() { - List<Exception> exceptions = new ArrayList<>(); - for (IOThread ioThread : ioThreads) { - try { - ioThread.close(); - } catch (Exception e) { - exceptions.add(e); - } - } - if (exceptions.isEmpty()) { - return; - } - if (exceptions.size() == 1) { - if (exceptions.get(0) instanceof RuntimeException) { - throw (RuntimeException) exceptions.get(0); - } else { - throw new RuntimeException(exceptions.get(0)); - } - } - StringBuilder b = new StringBuilder(); - b.append("Exception thrown while closing one or more endpoints: "); - for (int i = 0; i < exceptions.size(); i++) { - Exception e = exceptions.get(i); - b.append(Exceptions.toMessageString(e)); - if (i != (exceptions.size() - 1)) { - b.append(", "); - } - } - throw new RuntimeException(b.toString(), exceptions.get(0)); - } - - public String getStatsAsJSon() throws IOException { - StringWriter stringWriter = new StringWriter(); - JsonGenerator jsonGenerator = jsonMapper.createGenerator(stringWriter); - jsonGenerator.writeStartObject(); - jsonGenerator.writeArrayFieldStart("session"); - for (IOThread ioThread : ioThreads) { - jsonGenerator.writeStartObject(); - jsonGenerator.writeObjectFieldStart("endpoint"); - jsonGenerator.writeStringField("host", ioThread.getEndpoint().getHostname()); - jsonGenerator.writeNumberField("port", ioThread.getEndpoint().getPort()); - jsonGenerator.writeEndObject(); - jsonGenerator.writeFieldName("stats"); - IOThread.ConnectionStats connectionStats = ioThread.getConnectionStats(); - jsonMapper.writeValue(jsonGenerator, connectionStats); - jsonGenerator.writeEndObject(); - } - jsonGenerator.writeEndArray(); - jsonGenerator.writeEndObject(); - jsonGenerator.close(); - return stringWriter.toString(); - } - - public List<IOThread> ioThreads() { - return Collections.unmodifiableList(ioThreads); - } - - @Override - public boolean equals(Object o) { - return (this == o) || (o instanceof ClusterConnection && clusterId == ((ClusterConnection) o).clusterId); - } - - @Override - public int hashCode() { - return clusterId; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java deleted file mode 100644 index 8164534ca37..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DocumentQueue.java +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.Document; - -import java.time.Clock; -import java.time.Duration; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -/** - * Shared document queue that gives clients operations on documents which do not have operations already in flight. - * This is multithread safe. - * - * @author dybis - */ -class DocumentQueue { - - private final Deque<Document> queue; - private final int maxSize; - private boolean closed = false; - private final Clock clock; - - DocumentQueue(int maxSize, Clock clock) { - this.maxSize = maxSize; - this.queue = new ArrayDeque<>(maxSize); - this.clock = clock; - } - - List<Document> removeAllDocuments() { - synchronized (queue) { - List<Document> allDocs = new ArrayList<>(); - while (!queue.isEmpty()) { - allDocs.add(queue.poll()); - } - queue.notifyAll(); - return allDocs; - } - } - - void put(Document document, boolean calledFromIoThreadGroup) throws InterruptedException { - document.setQueueInsertTime(clock.instant()); - synchronized (queue) { - while (!closed && (queue.size() >= maxSize) && !calledFromIoThreadGroup) { - queue.wait(); - } - if (closed) { - throw new IllegalStateException("Cannot add elements to closed queue."); - } - queue.add(document); - queue.notifyAll(); - } - } - - Document poll(long timeout, TimeUnit unit) throws InterruptedException { - synchronized (queue) { - long remainingToWait = unit.toMillis(timeout); - while (queue.isEmpty()) { - long startTime = clock.millis(); - queue.wait(remainingToWait); - remainingToWait -= (clock.millis() - startTime); - if (remainingToWait <= 0) { - break; - } - } - Document document = queue.poll(); - queue.notifyAll(); - return document; - } - } - - Document poll() { - synchronized (queue) { - Document document = queue.poll(); - queue.notifyAll(); - return document; - } - } - - boolean isEmpty() { - synchronized (queue) { - return queue.isEmpty(); - } - } - - int size() { - synchronized (queue) { - return queue.size(); - } - } - - void clear() { - synchronized (queue) { - queue.clear(); - queue.notifyAll(); - } - } - - boolean close() { - boolean previousState; - synchronized (queue) { - previousState = closed; - closed = true; - queue.notifyAll(); - } - return previousState; - } - - Optional<Document> pollDocumentIfTimedoutInQueue(Duration localQueueTimeOut) { - synchronized (queue) { - if (queue.isEmpty()) return Optional.empty(); - - Document document = queue.peek(); - if (document.getQueueInsertTime().plus(localQueueTimeOut).isBefore(clock.instant())) - return Optional.ofNullable(queue.poll()); - else - return Optional.empty(); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java deleted file mode 100644 index a6f6992b238..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnection.java +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.OperationStatus; -import com.yahoo.vespa.http.client.core.ServerResponseException; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Dummy implementation. - * - * @author dybis - */ -public class DryRunGatewayConnection implements GatewayConnection { - - private final Endpoint endpoint; - private final Clock clock; - private Instant connectionTime = null; - private Instant lastPollTime = null; - - /** Set to true to hold off responding with a result to any incoming operations until this is set false */ - private boolean hold = false; - private final List<Document> held = new ArrayList<>(); - - /** If this is set, handshake operations will throw this exception */ - private ServerResponseException throwThisOnHandshake = null; - - /** If this is set, all write operations will throw this exception */ - private IOException throwThisOnWrite = null; - - public DryRunGatewayConnection(Endpoint endpoint, Clock clock) { - this.endpoint = endpoint; - this.clock = clock; - } - - @Override - public synchronized InputStream write(List<Document> docs) throws IOException { - if (throwThisOnWrite != null) - throw throwThisOnWrite; - - if (hold) { - held.addAll(docs); - return new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)); - } - else { - StringBuilder result = new StringBuilder(); - for (Document doc : held) - result.append(okResponse(doc).render()); - held.clear(); - for (Document doc : docs) - result.append(okResponse(doc).render()); - return new ByteArrayInputStream(result.toString().getBytes(StandardCharsets.UTF_8)); - } - } - - @Override - public synchronized InputStream poll() throws IOException { - lastPollTime = clock.instant(); - return write(new ArrayList<>()); - } - - @Override - public synchronized Instant lastPollTime() { return lastPollTime; } - - @Override - public synchronized InputStream drain() throws IOException { - return write(new ArrayList<>()); - } - - @Override - public synchronized boolean connect() { - connectionTime = clock.instant(); - return true; - } - - @Override - public synchronized Instant connectionTime() { return connectionTime; } - - @Override - public Endpoint getEndpoint() { - return endpoint; - } - - @Override - public synchronized void handshake() throws ServerResponseException { - if (throwThisOnHandshake != null) - throw throwThisOnHandshake; - } - - @Override - public synchronized void close() { } - - public synchronized void hold(boolean hold) { - this.hold = hold; - } - - /** Returns the document currently held in this */ - public synchronized List<Document> held() { return Collections.unmodifiableList(held); } - - public synchronized void throwOnWrite(IOException throwThisOnWrite) { - this.throwThisOnWrite = throwThisOnWrite; - } - - public synchronized void throwOnHandshake(ServerResponseException throwThisOnHandshake) { - this.throwThisOnHandshake = throwThisOnHandshake; - } - - private OperationStatus okResponse(Document document) { - return new OperationStatus("ok", document.getOperationId(), ErrorCode.OK, false, ""); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnectionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnectionFactory.java deleted file mode 100644 index 01bec563889..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/DryRunGatewayConnectionFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.Endpoint; - -import java.time.Clock; - -/** - * @author bratseth - */ -public class DryRunGatewayConnectionFactory implements GatewayConnectionFactory { - - private final Endpoint endpoint; - private final Clock clock; - - public DryRunGatewayConnectionFactory(Endpoint endpoint, Clock clock) { - this.endpoint = endpoint; - this.clock = clock; - } - - @Override - public GatewayConnection newConnection() { - return new DryRunGatewayConnection(endpoint, clock); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java deleted file mode 100644 index 34b8ef76068..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointIOException.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.Endpoint; - -import java.io.IOException; - -/** - * Class for throwing exception from endpoint. - * - * @author dybis -*/ -public class EndpointIOException extends IOException { - - private final Endpoint endpoint; - private static final long serialVersionUID = 29335813211L; - - public EndpointIOException(Endpoint endpoint, String message, Throwable cause) { - super(message, cause); - this.endpoint = endpoint; - } - - /** Returns the endpoint, or null if the failure occurred before this was assigned to a unique endpoint */ - public Endpoint getEndpoint() { return endpoint; } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java deleted file mode 100644 index 98b6aee1e33..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueue.java +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.FeedEndpointException; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.operationProcessor.EndPointResultFactory; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -/** - * The shared queue of operation results. - * This is multithread safe. - * - * @author Einar M R Rosenvinge - */ -class EndpointResultQueue { - - private static final Logger log = Logger.getLogger(EndpointResultQueue.class.getName()); - private final OperationProcessor operationProcessor; - - private final Map<String, InflightOperation> inflightOperations = new HashMap<>(); - - private final Endpoint endpoint; - private final int clusterId; - private final ScheduledThreadPoolExecutor timer; - private final long totalTimeoutMs; - - EndpointResultQueue(OperationProcessor operationProcessor, - Endpoint endpoint, - int clusterId, - ScheduledThreadPoolExecutor timer, - long totalTimeoutMs) { - this.operationProcessor = operationProcessor; - this.endpoint = endpoint; - this.clusterId = clusterId; - this.timer = timer; - this.totalTimeoutMs = totalTimeoutMs; - } - - public synchronized void operationSent(String operationId, GatewayConnection connection) { - DocumentTimerTask task = new DocumentTimerTask(operationId); - ScheduledFuture<?> future = timer.schedule(task, totalTimeoutMs, TimeUnit.MILLISECONDS); - inflightOperations.put(operationId, new InflightOperation(future, connection)); - } - - public synchronized void failOperation(EndpointResult result, int clusterId) { - resultReceived(result, clusterId, false); - } - - public synchronized void resultReceived(EndpointResult result, int clusterId) { - resultReceived(result, clusterId, true); - } - - void onEndpointError(FeedEndpointException e) { - operationProcessor.onEndpointError(e); - } - - private synchronized void resultReceived(EndpointResult result, int clusterId, boolean duplicateGivesWarning) { - operationProcessor.resultReceived(result, clusterId); - InflightOperation operation = inflightOperations.remove(result.getOperationId()); - if (operation == null) { - if (duplicateGivesWarning) { - log.info("Result for ID '" + result.getOperationId() + "' received from '" + endpoint + - "', but we have no record of a sent operation. This may happen if an operation is " + - "initiated, but also retried, due to HTTP failure. Otherwise, something is wrong on " + - "the server side (bad VIP usage?), or operation was received _after_ client-side timeout."); - } - return; - } - operation.future.cancel(false); - } - - /** Called only from ScheduledThreadPoolExecutor thread in DocumentTimerTask.run(), see below */ - private synchronized void timeout(String operationId) { - InflightOperation operation = inflightOperations.remove(operationId); - if (operation == null) { - log.finer("Timeout of operation '" + operationId + "', but operation " + - "not found in map. Result was probably received just-in-time from server, while timeout " + - "task could not be cancelled."); - return; - } - EndpointResult endpointResult = EndPointResultFactory.createTransientError( - endpoint, operationId, new RuntimeException("Timed out waiting for reply from server.")); - operationProcessor.resultReceived(endpointResult, clusterId); - } - - public synchronized int getPendingSize() { - return inflightOperations.values().size(); - } - - public synchronized void failPending(Exception exception) { - inflightOperations.forEach((operationId, operation) -> { - operation.future.cancel(false); - EndpointResult result = EndPointResultFactory.createError(endpoint, operationId, exception); - operationProcessor.resultReceived(result, clusterId); - }); - inflightOperations.clear(); - } - - public synchronized boolean hasInflightOperations(GatewayConnection connection) { - return inflightOperations.entrySet().stream() - .anyMatch(entry -> entry.getValue().connection.equals(connection)); - } - - private class DocumentTimerTask implements Runnable { - - private final String operationId; - - private DocumentTimerTask(String operationId) { - this.operationId = operationId; - } - - @Override - public void run() { - timeout(operationId); - } - - } - - private static class InflightOperation { - final ScheduledFuture<?> future; - final GatewayConnection connection; - - InflightOperation(ScheduledFuture<?> future, GatewayConnection connection) { - this.future = future; - this.connection = connection; - } - } -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java deleted file mode 100644 index 25057a1dead..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnection.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import java.io.IOException; -import java.io.InputStream; -import java.time.Instant; -import java.util.List; - -public interface GatewayConnection { - - /** Returns the time this connected over the network, or null if not connected yet */ - Instant connectionTime(); - - /** Returns the last time poll was called on this, or null if never */ - Instant lastPollTime(); - - InputStream write(List<Document> docs) throws ServerResponseException, IOException; - - /** Returns any operation results that are ready now */ - InputStream poll() throws ServerResponseException, IOException; - - /** Attempt to drain all outstanding operations, even if this leads to blocking */ - InputStream drain() throws ServerResponseException, IOException; - - boolean connect(); - - Endpoint getEndpoint(); - - void handshake() throws ServerResponseException, IOException; - - void close(); - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnectionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnectionFactory.java deleted file mode 100644 index 4988b73510f..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayConnectionFactory.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -/** - * Creates gateway connections on request - * - * @author bratseth - */ -public interface GatewayConnectionFactory { - - GatewayConnection newConnection(); - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java deleted file mode 100644 index 41a1b6a7e87..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottler.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import java.util.concurrent.ThreadLocalRandom; - -/** - * When the gateways says it can not handle more load, we should send less load. That is the responsibility - * of this component - * - * @author dybis - */ -public class GatewayThrottler { - - private long backOffTimeMs = 0; - private final long maxSleepTimeMs; - - public GatewayThrottler(long maxSleepTimeMs) { - this.maxSleepTimeMs = maxSleepTimeMs; - } - - public void handleCall(int transientErrors) { - if (transientErrors > 0) { - backOffTimeMs = Math.min(maxSleepTimeMs, backOffTimeMs + distribute(100)); - } else { - backOffTimeMs = Math.max(0, backOffTimeMs - distribute(10)); - } - sleepMs(backOffTimeMs); - } - - protected void sleepMs(long sleepTime) { - try { - if (backOffTimeMs > 0L) { - Thread.sleep(backOffTimeMs); - } - } catch (InterruptedException e) { - // Do nothing - } - } - - public int distribute(int expected) { - double factor = 0.5 + ThreadLocalRandom.current().nextDouble(); - Double result = expected * factor; - return result.intValue(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java deleted file mode 100644 index aeb164227f1..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/IOThread.java +++ /dev/null @@ -1,649 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.FeedConnectException; -import com.yahoo.vespa.http.client.FeedProtocolException; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.Exceptions; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import com.yahoo.vespa.http.client.core.operationProcessor.EndPointResultFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.time.Clock; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Random; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Thread which feeds document operations asynchronously and processes the results. - * - * @author Einar M R Rosenvinge - */ -public class IOThread implements Runnable, AutoCloseable { - - private static final Logger log = Logger.getLogger(IOThread.class.getName()); - - private final Endpoint endpoint; - private final GatewayConnectionFactory connectionFactory; - private final DocumentQueue documentQueue; - private final EndpointResultQueue resultQueue; - - /** The thread running this, or null if it does not run a thread (meaning tick() must be called from the outside) */ - private final Thread thread; - private final int clusterId; - private final CountDownLatch running = new CountDownLatch(1); - private final CountDownLatch stopSignal = new CountDownLatch(1); - private final int maxChunkSizeBytes; - private final int maxInFlightRequests; - private final Duration localQueueTimeOut; - private final GatewayThrottler gatewayThrottler; - private final Duration connectionTimeToLive; - private final long pollIntervalUS; - private final Clock clock; - private final Random random = new Random(); - private final OldConnectionsDrainer oldConnectionsDrainer; - - private volatile GatewayConnection currentConnection; - private volatile ConnectionState connectionState = ConnectionState.DISCONNECTED; - - private enum ConnectionState { DISCONNECTED, CONNECTED, SESSION_SYNCED }; - private final AtomicInteger wrongSessionDetectedCounter = new AtomicInteger(0); - private final AtomicInteger wrongVersionDetectedCounter = new AtomicInteger(0); - private final AtomicInteger problemStatusCodeFromServerCounter = new AtomicInteger(0); - private final AtomicInteger executeProblemsCounter = new AtomicInteger(0); - private final AtomicInteger docsReceivedCounter = new AtomicInteger(0); - private final AtomicInteger statusReceivedCounter = new AtomicInteger(0); - private final AtomicInteger pendingDocumentStatusCount = new AtomicInteger(0); - private final AtomicInteger successfulHandshakes = new AtomicInteger(0); - private final AtomicInteger lastGatewayProcessTimeMillis = new AtomicInteger(0); - - IOThread(ThreadGroup ioThreadGroup, - Endpoint endpoint, - EndpointResultQueue endpointResultQueue, - GatewayConnectionFactory connectionFactory, - int clusterId, - int maxChunkSizeBytes, - int maxInFlightRequests, - Duration localQueueTimeOut, - DocumentQueue documentQueue, - long maxSleepTimeMs, - Duration connectionTimeToLive, - boolean runThreads, - double idlePollFrequency, - Clock clock) { - this.endpoint = endpoint; - this.documentQueue = documentQueue; - this.connectionFactory = connectionFactory; - this.currentConnection = connectionFactory.newConnection(); - this.resultQueue = endpointResultQueue; - this.clusterId = clusterId; - this.maxChunkSizeBytes = maxChunkSizeBytes; - this.maxInFlightRequests = maxInFlightRequests; - this.connectionTimeToLive = connectionTimeToLive; - this.gatewayThrottler = new GatewayThrottler(maxSleepTimeMs); - this.pollIntervalUS = Math.max(1000, (long)(1000000.0/Math.max(0.1, idlePollFrequency))); // ensure range [1ms, 10s] - this.clock = clock; - this.localQueueTimeOut = localQueueTimeOut; - this.oldConnectionsDrainer = new OldConnectionsDrainer(endpoint, - clusterId, - Duration.ofMillis(pollIntervalUS/1000), - connectionTimeToLive, - localQueueTimeOut, - statusReceivedCounter, - resultQueue, - stopSignal, - clock); - if (runThreads) { - this.thread = new Thread(ioThreadGroup, this, "IOThread " + endpoint); - thread.setDaemon(true); - thread.start(); - Thread thread = new Thread(ioThreadGroup, oldConnectionsDrainer, "IOThread " + endpoint + " drainer"); - thread.setDaemon(true); - thread.start(); - } - else { - this.thread = null; - } - } - - public Endpoint getEndpoint() { - return endpoint; - } - - /** - * Returns a snapshot of counters. Threadsafe. - */ - public ConnectionStats getConnectionStats() { - return new ConnectionStats( - wrongSessionDetectedCounter.get(), - wrongVersionDetectedCounter.get(), - problemStatusCodeFromServerCounter.get(), - executeProblemsCounter.get(), - docsReceivedCounter.get(), - statusReceivedCounter.get(), - pendingDocumentStatusCount.get(), - successfulHandshakes.get(), - lastGatewayProcessTimeMillis.get()); - } - - @Override - public void close() { - documentQueue.close(); - if (stopSignal.getCount() == 0) return; - - stopSignal.countDown(); - log.finer("Closed called."); - - oldConnectionsDrainer.close(); - - // Make a last attempt to get results from previous operations, we have already waited quite a bit before getting here. - int size = resultQueue.getPendingSize(); - if (size > 0) { - log.info("We have outstanding operations (" + size + ") , trying to fetch responses."); - try { - processResponse(currentConnection.drain()); - } catch (Throwable e) { - log.log(Level.SEVERE, "Some failures while trying to get latest responses from vespa.", e); - } - } - - try { - currentConnection.close(); - } finally { - // If there is still documents in the queue, fail them. - drainDocumentQueueWhenFailingPermanently(new Exception("Closed call, did not manage to process everything so failing this document.")); - } - - log.fine("Session to " + endpoint + " closed."); - } - - /** For testing only */ - public void post(Document document) throws InterruptedException { - documentQueue.put(document, true); - } - - @Override - public String toString() { - return "I/O thread (for " + endpoint + ")"; - } - - List<Document> getNextDocsForFeeding(long maxWaitUnits, TimeUnit timeUnit) { - List<Document> docsForSendChunk = new ArrayList<>(); - int chunkSizeBytes = 0; - try { - drainFirstDocumentsInQueueIfOld(); - Document doc = thread != null ? documentQueue.poll(maxWaitUnits, timeUnit) : documentQueue.poll(); - if (doc != null) { - docsForSendChunk.add(doc); - chunkSizeBytes = doc.size(); - } - } catch (InterruptedException ie) { - log.fine("Got break signal while waiting for new documents to feed"); - return docsForSendChunk; - } - int pendingSize = 1 + resultQueue.getPendingSize(); - - // see if we can get more documents without blocking - // slightly randomize how much is taken to avoid harmonic interactions leading - // to some threads consistently taking more than others - int thisMaxChunkSizeBytes = randomize(maxChunkSizeBytes); - int thisMaxInFlightRequests = randomize(maxInFlightRequests); - while (chunkSizeBytes < thisMaxChunkSizeBytes && pendingSize < thisMaxInFlightRequests) { - drainFirstDocumentsInQueueIfOld(); - Document document = documentQueue.poll(); - if (document == null) break; - docsForSendChunk.add(document); - chunkSizeBytes += document.size(); - pendingSize++; - } - if (log.isLoggable(Level.FINEST)) - log.finest("Chunk has " + docsForSendChunk.size() + " docs with a size " + chunkSizeBytes + " bytes"); - docsReceivedCounter.addAndGet(docsForSendChunk.size()); - return docsForSendChunk; - } - - private int randomize(int limit) { - double multiplier = 0.75 + 0.25 * random.nextDouble(); - return Math.max(1, (int)(limit * multiplier)); - } - - private void addDocumentsToResultQueue(List<Document> docs) { - for (Document doc : docs) { - resultQueue.operationSent(doc.getOperationId(), currentConnection); - } - } - - private void markDocumentAsFailed(List<Document> docs, ServerResponseException servletException) { - for (Document doc : docs) { - resultQueue.failOperation(EndPointResultFactory.createTransientError(endpoint, - doc.getOperationId(), - servletException), - clusterId); - } - } - - private InputStream sendAndReceive(List<Document> docs) throws IOException, ServerResponseException { - try { - // Post the new docs and get async responses for other posts. - return currentConnection.write(docs); - } catch (ServerResponseException ser) { - markDocumentAsFailed(docs, ser); - throw ser; - } catch (Exception e) { - markDocumentAsFailed(docs, new ServerResponseException(Exceptions.toMessageString(e))); - throw e; - } - } - - private static class ProcessResponse { - - private final int transitiveErrorCount; - private final int processResultsCount; - - ProcessResponse(int transitiveErrorCount, int processResultsCount) { - this.transitiveErrorCount = transitiveErrorCount; - this.processResultsCount = processResultsCount; - } - - } - - private ProcessResponse processResponse(InputStream serverResponse) throws IOException { - return processResponse(serverResponse, endpoint, clusterId, statusReceivedCounter, resultQueue); - } - - private static ProcessResponse processResponse(InputStream serverResponse, - Endpoint endpoint, - int clusterId, - AtomicInteger statusReceivedCounter, - EndpointResultQueue resultQueue) throws IOException { - Collection<EndpointResult> endpointResults = EndPointResultFactory.createResult(endpoint, serverResponse); - statusReceivedCounter.addAndGet(endpointResults.size()); - int transientErrors = 0; - for (EndpointResult endpointResult : endpointResults) { - if (endpointResult.getDetail().getResultType() == Result.ResultType.TRANSITIVE_ERROR) { - transientErrors++; - } - resultQueue.resultReceived(endpointResult, clusterId); - } - return new ProcessResponse(transientErrors, endpointResults.size()); - } - - private ProcessResponse feedDocumentAndProcessResults(List<Document> docs) - throws ServerResponseException, IOException { - addDocumentsToResultQueue(docs); - long startTime = clock.millis(); - InputStream serverResponse = sendAndReceive(docs); - - ProcessResponse processResponse = processResponse(serverResponse); - lastGatewayProcessTimeMillis.set((int) (clock.millis() - startTime)); - return processResponse; - } - - private ProcessResponse pullAndProcessData(long maxWaitTimeUS) throws ServerResponseException, IOException { - int pendingResultQueueSize = resultQueue.getPendingSize(); - pendingDocumentStatusCount.set(pendingResultQueueSize); - - List<Document> nextDocsForFeeding = (pendingResultQueueSize > maxInFlightRequests) - ? new ArrayList<>() // The queue is full, will not send more documents - : getNextDocsForFeeding(maxWaitTimeUS, TimeUnit.MICROSECONDS); - - if (nextDocsForFeeding.isEmpty() && pendingResultQueueSize == 0) { - //we have no unfinished business with the server now. - log.finest("No document awaiting feeding, not waiting for results."); - return new ProcessResponse(0, 0); - } - log.finest("Awaiting " + pendingResultQueueSize + " results."); - ProcessResponse processResponse = feedDocumentAndProcessResults(nextDocsForFeeding); - - if (pendingResultQueueSize > maxInFlightRequests && processResponse.processResultsCount == 0) { - try { - // Max outstanding document operations, no more results on server side, wait a bit before asking again - Thread.sleep(300); - } catch (InterruptedException e) { - // Ignore - } - } - return processResponse; - } - - /** Given a current connection state, take the appropriate action and return the resulting new connection state */ - private ConnectionState cycle(ConnectionState connectionState) { - switch(connectionState) { - case DISCONNECTED: - try { - if (! currentConnection.connect()) { - log.log(Level.WARNING, "Could not connect to endpoint: '" + endpoint + "'. Will re-try."); - drainFirstDocumentsInQueueIfOld(); - return ConnectionState.DISCONNECTED; - } - return ConnectionState.CONNECTED; - } catch (Throwable throwable1) { - drainFirstDocumentsInQueueIfOld(); - - log.log(Level.INFO, "Failed connecting to endpoint: '" + endpoint + "'. Will re-try connecting.", - throwable1); - executeProblemsCounter.incrementAndGet(); - return ConnectionState.DISCONNECTED; - } - case CONNECTED: - try { - if (isStale(currentConnection)) - return refreshConnection(connectionState); - currentConnection.handshake(); - successfulHandshakes.getAndIncrement(); - } catch (ServerResponseException ser) { - int code = ser.getResponseCode(); - if (code == 401 || code == 403) { - drainDocumentQueueWhenFailingPermanently(new Exception("Denied access by endpoint:" + ser.getResponseString())); - log.log(Level.SEVERE, "Failed authentication or authorization with '" + endpoint + "': " + Exceptions.toMessageString(ser)); - return ConnectionState.CONNECTED; // Should ideally exit immediately, instead of doing this per X documents :/ - } - - executeProblemsCounter.incrementAndGet(); - log.log(Level.INFO, "Failed talking to endpoint. Handshake with server endpoint '" + endpoint + - "' failed -- will re-try handshake: " + Exceptions.toMessageString(ser)); - - drainFirstDocumentsInQueueIfOld(); - resultQueue.onEndpointError(new FeedProtocolException(ser.getResponseCode(), ser.getResponseString(), ser, endpoint)); - return ConnectionState.CONNECTED; - } catch (Throwable throwable) { // This cover IOException as well - executeProblemsCounter.incrementAndGet(); - resultQueue.onEndpointError(new FeedConnectException(throwable, endpoint)); - log.log(Level.INFO, "Failed talking to endpoint. Handshake with server endpoint '" + - endpoint + "' failed. Will re-try handshake.", - throwable); - drainFirstDocumentsInQueueIfOld(); - currentConnection.close(); - return ConnectionState.DISCONNECTED; - } - return ConnectionState.SESSION_SYNCED; - case SESSION_SYNCED: - try { - if (isStale(currentConnection)) - return refreshConnection(connectionState); - ProcessResponse processResponse = pullAndProcessData(pollIntervalUS); - gatewayThrottler.handleCall(processResponse.transitiveErrorCount); - } - catch (ServerResponseException ser) { - log.log(Level.INFO, "Problems while handing data over to endpoint '" + endpoint + - "'. Will re-try. Endpoint responded with an unexpected HTTP response code.", - ser); - return ConnectionState.CONNECTED; - } - catch (Throwable e) { - log.log(Level.INFO, - "Connection level error handing data over to endpoint '" + endpoint + "'. Will re-try.", - e); - currentConnection.close(); - return ConnectionState.DISCONNECTED; - } - return ConnectionState.SESSION_SYNCED; - default: { - log.severe("Should never get here."); - currentConnection.close(); - return ConnectionState.DISCONNECTED; - } - } - } - - private void sleepIfProblemsGettingSyncedConnection(ConnectionState newState, ConnectionState oldState) { - if (newState == ConnectionState.SESSION_SYNCED) return; - if (newState == ConnectionState.CONNECTED && oldState == ConnectionState.DISCONNECTED) return; - try { - // Take it easy we have problems getting a connection up. - if (stopSignal.getCount() > 0 || !documentQueue.isEmpty()) { - Thread.sleep(gatewayThrottler.distribute(3000)); - } - } catch (InterruptedException e) { - } - } - - @Override - public void run() { - while (stopSignal.getCount() > 0 || !documentQueue.isEmpty()) - tick(); - log.finer(toString() + " exiting, documentQueue.size()=" + documentQueue.size()); - running.countDown(); - } - - /** Do one iteration of work. Should be called from the single worker thread of this. */ - public void tick() { - ConnectionState oldState = connectionState; - connectionState = cycle(connectionState); - if (thread == null) - oldConnectionsDrainer.checkOldConnections(); - if (thread != null) - sleepIfProblemsGettingSyncedConnection(connectionState, oldState); - } - - private void drainFirstDocumentsInQueueIfOld() { - while (true) { - Optional<Document> document = documentQueue.pollDocumentIfTimedoutInQueue(localQueueTimeOut); - if ( ! document.isPresent()) return; - - EndpointResult endpointResult = EndPointResultFactory.createTransientError( - endpoint, document.get().getOperationId(), - new Exception("Not sending document operation, timed out in queue after " + - (clock.millis() - document.get().getQueueInsertTime().toEpochMilli()) + " ms.")); - resultQueue.failOperation(endpointResult, clusterId); - } - } - - private void drainDocumentQueueWhenFailingPermanently(Exception exception) { - // first, clear sentOperations: - resultQueue.failPending(exception); - - for (Document document : documentQueue.removeAllDocuments()) { - EndpointResult endpointResult= - EndPointResultFactory.createError(endpoint, document.getOperationId(), exception); - resultQueue.failOperation(endpointResult, clusterId); - } - } - - private boolean isStale(GatewayConnection connection) { - return connection.connectionTime() != null - && connection.connectionTime().plus(connectionTimeToLive).isBefore(clock.instant()); - } - - private ConnectionState refreshConnection(ConnectionState currentConnectionState) { - if (currentConnectionState == ConnectionState.SESSION_SYNCED) - oldConnectionsDrainer.add(currentConnection); - currentConnection = connectionFactory.newConnection(); - return ConnectionState.DISCONNECTED; - } - - public static class ConnectionStats { - - // NOTE: These fields are accessed by reflection in JSON serialization - - public final int wrongSessionDetectedCounter; - public final int wrongVersionDetectedCounter; - public final int problemStatusCodeFromServerCounter; - public final int executeProblemsCounter; - public final int docsReceivedCounter; - public final int statusReceivedCounter; - public final int pendingDocumentStatusCount; - public final int successfullHandshakes; - public final int lastGatewayProcessTimeMillis; - - ConnectionStats(int wrongSessionDetectedCounter, - int wrongVersionDetectedCounter, - int problemStatusCodeFromServerCounter, - int executeProblemsCounter, - int docsReceivedCounter, - int statusReceivedCounter, - int pendingDocumentStatusCount, - int successfullHandshakes, - int lastGatewayProcessTimeMillis) { - this.wrongSessionDetectedCounter = wrongSessionDetectedCounter; - this.wrongVersionDetectedCounter = wrongVersionDetectedCounter; - this.problemStatusCodeFromServerCounter = problemStatusCodeFromServerCounter; - this.executeProblemsCounter = executeProblemsCounter; - this.docsReceivedCounter = docsReceivedCounter; - this.statusReceivedCounter = statusReceivedCounter; - this.pendingDocumentStatusCount = pendingDocumentStatusCount; - this.successfullHandshakes = successfullHandshakes; - this.lastGatewayProcessTimeMillis = lastGatewayProcessTimeMillis; - } - } - - /** For testing. Returns the current connection of this. Not thread safe. */ - public GatewayConnection currentConnection() { return currentConnection; } - - /** For testing. Returns a snapshot of the old connections of this. */ - public List<GatewayConnection> oldConnections() { return oldConnectionsDrainer.connections(); } - - /** For testing */ - public EndpointResultQueue resultQueue() { return resultQueue; } - - /** - * We need to drain results on the connection where they were sent to make sure we request results on - * the node which received the operation also when going through a VIP. - */ - private static class OldConnectionsDrainer implements Runnable { - - private static final Logger log = Logger.getLogger(OldConnectionsDrainer.class.getName()); - - private final Endpoint endpoint; - private final int clusterId; - private final Duration pollInterval; - private final Duration connectionTimeToLive; - private final Duration localQueueTimeOut; - private final AtomicInteger statusReceivedCounter; - private final EndpointResultQueue resultQueue; - private final CountDownLatch stopSignal; - private final Clock clock; - - /** - * Previous connections on which we may have sent operations and are still waiting for the results - * All connections in this are in state SESSION_SYNCED. - */ - private final List<GatewayConnection> connections = new CopyOnWriteArrayList<>(); - - OldConnectionsDrainer(Endpoint endpoint, - int clusterId, - Duration pollInterval, - Duration connectionTimeToLive, - Duration localQueueTimeOut, - AtomicInteger statusReceivedCounter, - EndpointResultQueue resultQueue, - CountDownLatch stopSignal, - Clock clock) { - this.endpoint = endpoint; - this.clusterId = clusterId; - this.pollInterval = pollInterval; - this.connectionTimeToLive = connectionTimeToLive; - this.localQueueTimeOut = localQueueTimeOut; - this.statusReceivedCounter = statusReceivedCounter; - this.resultQueue = resultQueue; - this.stopSignal = stopSignal; - this.clock = clock; - } - - /** Add another old connection to this for draining */ - public void add(GatewayConnection connection) { - connections.add(connection); - } - - @Override - public void run() { - while (stopSignal.getCount() > 0) { - try { - checkOldConnections(); - Thread.sleep(pollInterval.toMillis()); - } - catch (InterruptedException e) { - log.log(Level.WARNING, "Close thread was interrupted: " + e.getMessage(), e); - Thread.currentThread().interrupt(); - return; - } catch (Exception e) { - log.log(Level.WARNING, "Connection draining failed: " + e.getMessage(), e); - } - } - } - - public void checkOldConnections() { - for (GatewayConnection connection : connections) { - if (!resultQueue.hasInflightOperations(connection)) { - log.fine(() -> connection + " no longer has inflight operations"); - closeConnection(connection); - } else if (closingTime(connection).isBefore(clock.instant())) { - log.fine(() -> connection + " still has inflight operations, but drain period is over"); - tryPollAndDrainInflightOperations(connection); - closeConnection(connection); - } else if (timeToPoll(connection)) { - tryPollAndDrainInflightOperations(connection); - } - } - } - - private void closeConnection(GatewayConnection connection) { - log.fine(() -> "Closing " + connection); - connection.close(); - connections.remove(connection); // Safe as CopyOnWriteArrayList allows removal during iteration - } - - private void tryPollAndDrainInflightOperations(GatewayConnection connection) { - try { - log.fine(() -> "Polling and draining inflight operations for " + connection); - IOThread.processResponse(connection.poll(), endpoint, clusterId, statusReceivedCounter, resultQueue); - } catch (Exception e) { - // Old connection; best effort - log.log(Level.FINE, e, () -> "Polling status of inflight operations failed: " + e.getMessage()); - } - } - - private boolean timeToPoll(GatewayConnection connection) { - // connectionEndOfLife < connectionLastPolled < now - Instant now = clock.instant(); - Instant endOfLife = connection.connectionTime().plus(connectionTimeToLive); - if (connection.lastPollTime() == null) return endOfLife.plus(pollInterval).isBefore(now); - if (connection.lastPollTime().plus(pollInterval).isAfter(now)) return false; - - // Exponential (2^x) dropoff: - double connectionEndOfLife = endOfLife.toEpochMilli(); - double connectionLastPolled = connection.lastPollTime().toEpochMilli(); - return now.toEpochMilli() - connectionEndOfLife > 2 * (connectionLastPolled - connectionEndOfLife); - } - - private Instant closingTime(GatewayConnection connection) { - return connection.connectionTime().plus(connectionTimeToLive).plus(localQueueTimeOut); - } - - private void close() { - int size = resultQueue.getPendingSize(); - if (size > 0) { - log.info("We have outstanding operations (" + size + ") , trying to fetch responses."); - for (GatewayConnection connection : connections) { - try { - IOThread.processResponse(connection.poll(), endpoint, clusterId, statusReceivedCounter, resultQueue); - } catch (Throwable e) { - log.log(Level.SEVERE, "Some failures while trying to get latest responses from vespa.", e); - } - } - } - for (GatewayConnection oldConnection : connections) - oldConnection.close(); - } - - /** For testing. Returns the old connections of this. */ - public List<GatewayConnection> connections() { return Collections.unmodifiableList(connections); } - - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java deleted file mode 100644 index 79a04d8f043..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlocker.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import java.util.concurrent.Semaphore; - -/** - * A semaphore that can be re-sized. - * - * @author dybis - */ -final public class ConcurrentDocumentOperationBlocker { - - private static final int INITIAL_SIZE = 0; - private final ReducableSemaphore semaphore = new ReducableSemaphore(); - private int maxConcurrency = INITIAL_SIZE; - private final Object monitor = new Object(); - - /* - * Resizes the semaphore. It does not wait for threads that are in the queue when downsizing. - */ - void setMaxConcurrency(int maxConcurrency) { - synchronized (monitor) { - int deltaConcurrency = maxConcurrency - this.maxConcurrency; - - if (deltaConcurrency > 0) { - semaphore.release(deltaConcurrency); - } - if (deltaConcurrency < 0) { - semaphore.reducePermits(-1 * deltaConcurrency); - } - this.maxConcurrency = maxConcurrency; - } - } - - /** - * Release a permit. - */ - void operationDone() { - semaphore.release(); - } - - /** - * Acquire a permit. Blocking if no permits available. - */ - void startOperation() throws InterruptedException { - semaphore.acquire(); - } - - int availablePermits() { - return semaphore.availablePermits(); - } - - /** - * We need to extend Semaphore to get access to protected reducePermit() method. - */ - @SuppressWarnings("serial") - private static final class ReducableSemaphore extends Semaphore { - - ReducableSemaphore() { - super(INITIAL_SIZE, true /* FIFO */); - } - - @Override - protected void reducePermits(int reduction) { - super.reducePermits(reduction); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java deleted file mode 100644 index b72a6c67398..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/DocumentSendInfo.java +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.core.Document; - -import java.time.Clock; -import java.util.HashMap; -import java.util.Map; - -/** - * Keeps an overview of what is sent and what is received for an operation. - * This class is not thread-safe. - */ -class DocumentSendInfo { - - private final Document document; - private final Map<Integer, Result.Detail> detailByClusterId = new HashMap<>(); - // This is lazily populated as normal cases does not require retries. - private Map<Integer, Integer> attemptedRetriesByClusterId = null; - private final StringBuilder localTrace; - private final Clock clock; - - DocumentSendInfo(Document document, boolean traceThisDoc, Clock clock) { - this.document = document; - localTrace = traceThisDoc ? new StringBuilder("\n" + document.createTime() + " Trace starting " + "\n") - : null; - this.clock = clock; - } - - boolean addIfNotAlreadyThere(Result.Detail detail, int clusterId) { - if (detailByClusterId.containsKey(clusterId)) { - if (localTrace != null) { - localTrace.append(clock.millis() + " Got duplicate detail, ignoring this: " + - detail.toString() + "\n"); - } - return false; - } - if (localTrace != null) { - localTrace.append(clock.millis() + " Got detail: " + detail.toString() + "\n"); - } - detailByClusterId.put(clusterId, detail); - return true; - } - - int detailCount() { - return detailByClusterId.size(); - } - - public Result createResult() { - return new Result(document, detailByClusterId.values(), localTrace); - } - - int incRetries(int clusterId, Result.Detail detail) { - if (attemptedRetriesByClusterId == null) { - attemptedRetriesByClusterId = new HashMap<>(); - } - int retries = 0; - if (attemptedRetriesByClusterId.containsKey(clusterId)) { - retries = attemptedRetriesByClusterId.get(clusterId); - } - retries++; - attemptedRetriesByClusterId.put(clusterId, retries); - if (localTrace != null) { - localTrace.append(clock.millis() + " Asked about retrying for cluster ID " - + clusterId + ", number of retries is " + retries + " Detail:\n" + detail.toString()); - } - return retries; - } - - Document getDocument() { - return document; - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java deleted file mode 100644 index 9af63f6637a..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/EndPointResultFactory.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.OperationStatus; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.logging.Logger; - -/** - * @author Einar M R Rosenvinge - */ -public final class EndPointResultFactory { - - private static final Logger log = Logger.getLogger(EndPointResultFactory.class.getName()); - private static final String EMPTY_MESSAGE = "-"; - - public static Collection<EndpointResult> createResult(Endpoint endpoint, - InputStream inputStream) throws IOException { - List<EndpointResult> results = new ArrayList<>(); - try (BufferedReader reader = new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.US_ASCII))) { - String line; - while ((line = reader.readLine()) != null) { - results.add(parseResult(line, endpoint)); - } - } - return results; - } - - public static EndpointResult createError(Endpoint endpoint, String operationId, Exception exception) { - return new EndpointResult(operationId, new Result.Detail(endpoint, - Result.ResultType.FATAL_ERROR, - null, - exception)); - } - - public static EndpointResult createTransientError(Endpoint endpoint, String operationId, Exception exception) { - return new EndpointResult(operationId, new Result.Detail(endpoint, - Result.ResultType.TRANSITIVE_ERROR, - null, - exception)); - } - - private static Result.ResultType replyToResultType(OperationStatus reply) { - // The ordering below is important, e.g. if success, it is never a transient error even if isTransient is true. - if (reply.errorCode.isSuccess()) - return Result.ResultType.OPERATION_EXECUTED; - if (reply.isConditionNotMet) - return Result.ResultType.CONDITION_NOT_MET; - if (reply.errorCode.isTransient()) - return Result.ResultType.TRANSITIVE_ERROR; - return Result.ResultType.FATAL_ERROR; - } - - private static EndpointResult parseResult(String line, Endpoint endpoint) { - try { - OperationStatus reply = OperationStatus.parse(line); - String message; - if (EMPTY_MESSAGE.equals(reply.message)) { - message = null; - } else { - message = reply.message; - } - Exception exception = null; - if (!reply.errorCode.isSuccess() && message != null) { - exception = new RuntimeException(message); - } - if (reply.traceMessage != null && !reply.traceMessage.isEmpty()) { - log.fine("Got trace message: " + reply.traceMessage); - } - return new EndpointResult( - reply.operationId, - new Result.Detail(endpoint, - replyToResultType(reply), - reply.traceMessage, - exception)); - } catch (Throwable t) { - throw new IllegalArgumentException("Bad result line from server: '" + line + "'", t); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java deleted file mode 100644 index a88d5af8094..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottler.java +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.core.ThrottlePolicy; - -import java.time.Clock; -import java.util.concurrent.ThreadLocalRandom; - -/** - * Adjusts in-flight operations based on throughput. It will walk the graph and try to find - * local optimum. - * - * It looks at the throughput, adjust max in-flight based on the previous throughput and settings. - * - * In the beginning it moves faster, and then stabilizes. - * - * It will wait a bit after adjusting before it starts to sample, since there is a latency between adjustment - * and result. - * - * There are several mechanisms to reduce impact of several clients running in parallel. The window size has a - * random part, and the wait time before sampling after adjustment has a random part as well. - * - * To avoid running wild with large values of max-in flight, it is tuned to stay on the smaller part, and - * rather reduce max-in flight than to have a too large value. - * - * In case the where the queue is moved to minimum size, it will now and then increase queue size to get - * more sample data and possibly grow size. - * - * Class is fully thread safe, i.e. all public methods are thread safe. - * - * @author dybis - */ -public class IncompleteResultsThrottler { - - private final ConcurrentDocumentOperationBlocker blocker = new ConcurrentDocumentOperationBlocker(); - private final int maxInFlightValue; - private final int minInFlightValue; - private final ThrottlePolicy policy; - - // 9-11 seconds with some randomness to avoid fully synchronous feeders. - public final long phaseSizeMs = 9000 + (ThreadLocalRandom.current().nextInt() % 2000); - private final Clock clock; - - private final Object monitor = new Object(); - private long sampleStartTimeMs = 0; - private int previousNumOk = 0; - private int previousMaxInFlight = 0; - private int stabilizingPhasesLeft = 0; - private int adjustCycleCount = 0; - private int maxInFlightNow; - private int numOk = 0; - private int minWindowSizeCounter = 0; - private int minPermitsAvailable = 0; - - protected static int INITIAL_MAX_IN_FLIGHT_VALUE = 200; - protected static int SECOND_MAX_IN_FLIGHT_VALUE = 270; - private StringBuilder debugMessage = new StringBuilder(); - - /** - * Creates the throttler. - * - * @param minInFlightValue the throttler will never throttle beyond this limit. - * @param maxInFlightValue the throttler will never throttle above this limit. If zero, no limit. - * @param clock use to calculate window size. Can be null if minWindowSize and maxInFlightValue are equal. - * @param policy is the algorithm for finding next value of the number of in-flight documents operations. - */ - public IncompleteResultsThrottler(int minInFlightValue, int maxInFlightValue, Clock clock, ThrottlePolicy policy) { - this.maxInFlightValue = maxInFlightValue == 0 ? Integer.MAX_VALUE : maxInFlightValue; - this.minInFlightValue = minInFlightValue == 0 ? this.maxInFlightValue : minInFlightValue; - this.policy = policy; - this.clock = clock; - if (minInFlightValue != maxInFlightValue) { - this.sampleStartTimeMs = clock.millis(); - } - setNewSemaphoreSize(INITIAL_MAX_IN_FLIGHT_VALUE); - } - - public int availableCapacity() { - return blocker.availablePermits(); - } - - public void operationStart() { - try { - blocker.startOperation(); - } catch (InterruptedException e) { - // Ignore - } - if (maxInFlightValue != minInFlightValue) { - synchronized (monitor) { - adjustThrottling(); - } - } - } - - public String getDebugMessage() { - synchronized (monitor) { - return debugMessage.toString(); - } - } - - public void resultReady(boolean success) { - blocker.operationDone(); - if (!success) { - return; - } - synchronized (monitor) { - numOk++; - minPermitsAvailable = Math.min(minPermitsAvailable, blocker.availablePermits()); - } - } - - // Only for testing - protected int waitingThreads() { - synchronized (monitor) { - return maxInFlightNow - blocker.availablePermits(); - } - } - - private double getCeilingDifferencePerformance(int adjustCycle) { - // We want larger adjustments in the early phase. - if (adjustCycle > 10) { - return 0.7; - } - return 1.2; - } - - private void adjustCycle() { - adjustCycleCount++; - stabilizingPhasesLeft = adjustCycleCount < 5 ? 1 : 2 + ThreadLocalRandom.current().nextInt() % 2; - - double maxPerformanceChange = getCeilingDifferencePerformance(adjustCycleCount); - boolean messagesQueued = minPermitsAvailable < 2; - - int newMaxInFlight = policy.calcNewMaxInFlight( - maxPerformanceChange, numOk, previousNumOk, previousMaxInFlight, maxInFlightNow, messagesQueued); - debugMessage = new StringBuilder(); - debugMessage.append("previousMaxInFlight: " + previousMaxInFlight - + " maxInFlightNow: " + maxInFlightNow - + " numOk: " + numOk + " " + " previousOk: " + previousNumOk - + " new size is: " + newMaxInFlight); - previousMaxInFlight = maxInFlightNow; - previousNumOk = numOk; - - setNewSemaphoreSize(adjustCycleCount == 1 ? SECOND_MAX_IN_FLIGHT_VALUE : newMaxInFlight); - } - - private void adjustThrottling() { - if (clock.millis() < sampleStartTimeMs + phaseSizeMs) return; - - sampleStartTimeMs += phaseSizeMs; - - if (stabilizingPhasesLeft-- == 0) { - adjustCycle(); - } - numOk = 0; - this.minPermitsAvailable = maxInFlightNow; - } - - private int tryBoostingSizeIfMinValueOverSeveralCycles(final int size) { - if (size <= minInFlightValue) { - minWindowSizeCounter++; - } else { - minWindowSizeCounter = 0; - } - if (minWindowSizeCounter == 4) { - debugMessage.append(" (inc max in flight to get more data)"); - minWindowSizeCounter = 0; - return size + 10; - } - return size; - - } - - private void setNewSemaphoreSize(final int size) { - maxInFlightNow = - Math.max(minInFlightValue, Math.min( - tryBoostingSizeIfMinValueOverSeveralCycles(size), maxInFlightValue)); - blocker.setMaxConcurrency(maxInFlightNow); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java deleted file mode 100644 index 5d0e063efdf..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessor.java +++ /dev/null @@ -1,307 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.google.common.collect.ArrayListMultimap; -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedEndpointException; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.communication.EndpointIOException; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.Exceptions; -import com.yahoo.vespa.http.client.core.communication.ClusterConnection; - -import java.math.BigInteger; -import java.security.SecureRandom; -import java.time.Clock; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * Merges several endpointResult into one Result and does the callback. - * - * @author dybis - */ -public class OperationProcessor { - - private static final Logger log = Logger.getLogger(OperationProcessor.class.getName()); - private final Map<String, DocumentSendInfo> docSendInfoByOperationId = new LinkedHashMap<>(); - private final ArrayListMultimap<String, Document> blockedDocumentsByDocumentId = ArrayListMultimap.create(); - private final Set<String> inflightDocumentIds = new HashSet<>(); - private final int numDestinations; - private final FeedClient.ResultCallback resultCallback; - private final Object monitor = new Object(); - private final IncompleteResultsThrottler incompleteResultsThrottler; - // Position in the array is cluster ID. - private final List<ClusterConnection> clusters = new ArrayList<>(); - private final ScheduledThreadPoolExecutor timeoutExecutor; - private final OperationStats operationStats; - private final int maxRetries; - private final long minTimeBetweenRetriesMs; - private final Random random = new SecureRandom(); - private final int traceEveryXOperation; - private int traceCounter = 0; - private final boolean traceToStderr; - private final ThreadGroup ioThreadGroup; - private final String clientId = new BigInteger(130, random).toString(32); - private final Clock clock; - - public OperationProcessor(IncompleteResultsThrottler incompleteResultsThrottler, - FeedClient.ResultCallback resultCallback, - SessionParams sessionParams, - ScheduledThreadPoolExecutor timeoutExecutor, - Clock clock) { - this.numDestinations = sessionParams.getClusters().size(); - this.resultCallback = resultCallback; - this.incompleteResultsThrottler = incompleteResultsThrottler; - this.timeoutExecutor = timeoutExecutor; - this.ioThreadGroup = new ThreadGroup("operationprocessor"); - this.clock = clock; - - if (sessionParams.getClusters().isEmpty()) - throw new IllegalArgumentException("Cannot feed to 0 clusters."); - - for (Cluster cluster : sessionParams.getClusters()) { - if (cluster.getEndpoints().isEmpty()) - throw new IllegalArgumentException("Cannot feed to empty cluster."); - } - - for (int i = 0; i < sessionParams.getClusters().size(); i++) { - Cluster cluster = sessionParams.getClusters().get(i); - clusters.add(new ClusterConnection(this, - sessionParams.getFeedParams(), - sessionParams.getConnectionParams(), - cluster, - i, - sessionParams.getClientQueueSize() / sessionParams.getClusters().size(), - timeoutExecutor, - clock)); - } - operationStats = new OperationStats(sessionParams, clusters, incompleteResultsThrottler); - maxRetries = sessionParams.getConnectionParams().getMaxRetries(); - minTimeBetweenRetriesMs = sessionParams.getConnectionParams().getMinTimeBetweenRetriesMs(); - traceEveryXOperation = sessionParams.getConnectionParams().getTraceEveryXOperation(); - traceToStderr = sessionParams.getConnectionParams().getPrintTraceToStdErr(); - } - - public ThreadGroup getIoThreadGroup() { - return ioThreadGroup; - } - - public int getIncompleteResultQueueSize() { - synchronized (monitor) { - return docSendInfoByOperationId.size(); - } - } - - /** Returns the id of the oldest operation to be sent. */ - public Optional<String> oldestIncompleteResultId() { - synchronized (monitor) { - return docSendInfoByOperationId.isEmpty() - ? Optional.empty() - : Optional.of(docSendInfoByOperationId.keySet().iterator().next()); - } - } - - public String getClientId() { - return clientId; - } - - private boolean retriedThis(EndpointResult endpointResult, DocumentSendInfo documentSendInfo, int clusterId) { - Result.Detail detail = endpointResult.getDetail(); - if (detail.getResultType() == Result.ResultType.OPERATION_EXECUTED) return false; // Success: No retries - - int retries = documentSendInfo.incRetries(clusterId, detail); - if (retries > maxRetries) return false; - - String exceptionMessage = detail.getException() == null ? "" : detail.getException().getMessage(); - if (exceptionMessage == null) - exceptionMessage = ""; - - // TODO: Return proper error code in structured data in next version of internal API. - // Error codes from messagebus/src/cpp/messagebus/errorcode.h - boolean retryThisOperation = - detail.getResultType() == Result.ResultType.TRANSITIVE_ERROR || - exceptionMessage.contains("SEND_QUEUE_CLOSED") || - exceptionMessage.contains("ILLEGAL_ROUTE") || - exceptionMessage.contains("NO_SERVICES_FOR_ROUTE") || - exceptionMessage.contains("NETWORK_ERROR") || - exceptionMessage.contains("SEQUENCE_ERROR") || - exceptionMessage.contains("NETWORK_SHUTDOWN") || - exceptionMessage.contains("TIMEOUT"); - - if (retryThisOperation) { - int waitTime = (int) (minTimeBetweenRetriesMs * (1 + random.nextDouble() / 3)); - log.finest("Retrying due to " + detail + " attempt " + retries + " in " + waitTime + " ms."); - timeoutExecutor.schedule(() -> postToCluster(clusters.get(clusterId), documentSendInfo.getDocument()), - waitTime, - TimeUnit.MILLISECONDS); - return true; - } - - return false; - } - - private Result process(EndpointResult endpointResult, int clusterId) { - Result result; - Document blockedDocumentToSend = null; - synchronized (monitor) { - if (!docSendInfoByOperationId.containsKey(endpointResult.getOperationId())) { - log.finer("Received out-of-order or too late result, discarding: " + endpointResult); - return null; - } - DocumentSendInfo documentSendInfo = docSendInfoByOperationId.get(endpointResult.getOperationId()); - - if (retriedThis(endpointResult, documentSendInfo, clusterId)) return null; - - // Duplicate message - if ( ! documentSendInfo.addIfNotAlreadyThere(endpointResult.getDetail(), clusterId)) return null; - - // Is this the last operation we are waiting for? - if (documentSendInfo.detailCount() != numDestinations) return null; - - result = documentSendInfo.createResult(); - docSendInfoByOperationId.remove(endpointResult.getOperationId()); - - String documentId = documentSendInfo.getDocument().getDocumentId(); - // If we got a pending operation against this document - // dont't remove it from inflightDocuments and send blocked document operation - List<Document> blockedDocuments = blockedDocumentsByDocumentId.get(documentId); - if (blockedDocuments.isEmpty()) { - inflightDocumentIds.remove(documentId); - } else { - blockedDocumentToSend = blockedDocuments.remove(0); - } - } - if (blockedDocumentToSend != null) { - sendToClusters(blockedDocumentToSend, clock); - } - return result; - } - - public void resultReceived(EndpointResult endpointResult, int clusterId) { - Result result = process(endpointResult, clusterId); - if (result != null) { - incompleteResultsThrottler.resultReady(result.isSuccess()); - resultCallback.onCompletion(result.getDocumentId(), result); - if (traceToStderr && result.hasLocalTrace()) { - System.err.println(result.toString()); - } - } - } - - public void onEndpointError(FeedEndpointException e) { - resultCallback.onEndpointException(e); - } - - public List<Exception> closeClusters() { - List<Exception> exceptions = new ArrayList<>(); - // first, close cluster sessions and allow connections to drain normally - for (ClusterConnection cluster : clusters) { - try { - cluster.close(); - } catch (Exception e) { - exceptions.add(e); - } - } - return exceptions; - } - - public void sendDocument(Document document) { - incompleteResultsThrottler.operationStart(); - - synchronized (monitor) { - if (inflightDocumentIds.contains(document.getDocumentId())) { - blockedDocumentsByDocumentId.put(document.getDocumentId(), document); - return; - } - inflightDocumentIds.add(document.getDocumentId()); - } - - sendToClusters(document, clock); - } - - private void sendToClusters(Document document, Clock clock) { - synchronized (monitor) { - boolean traceThisDoc = traceEveryXOperation > 0 && traceCounter++ % traceEveryXOperation == 0; - docSendInfoByOperationId.put(document.getOperationId(), new DocumentSendInfo(document, traceThisDoc, clock)); - } - - for (ClusterConnection clusterConnection : clusters) { - postToCluster(clusterConnection, document); - } - } - - private void postToCluster(ClusterConnection clusterConnection, Document document) { - try { - clusterConnection.post(document); - } catch (EndpointIOException eio) { - resultReceived(EndPointResultFactory.createError(eio.getEndpoint(), - document.getOperationId(), - eio), - clusterConnection.getClusterId()); - } - } - - public List<ClusterConnection> clusters() { return Collections.unmodifiableList(clusters); } - - public String getStatsAsJson() { - return operationStats.getStatsAsJson(); - } - - public void close() { - List<Exception> exceptions = closeClusters(); - try { - closeExecutor(); - } catch (InterruptedException e) { - exceptions.add(e); - } - - if (exceptions.isEmpty()) { - return; - } - if (exceptions.size() == 1) { - if (exceptions.get(0) instanceof RuntimeException) { - throw (RuntimeException) exceptions.get(0); - } else { - throw new RuntimeException(exceptions.get(0)); - } - } - - StringBuilder b = new StringBuilder(); - b.append("Exception thrown while closing one or more clusters: "); - for (int i = 0; i < exceptions.size(); i++) { - Exception e = exceptions.get(i); - b.append(Exceptions.toMessageString(e)); - if (i != (exceptions.size() - 1)) { - b.append(", "); - } - } - throw new RuntimeException(b.toString(), exceptions.get(0)); - } - - private void closeExecutor() throws InterruptedException { - log.log(Level.FINE, "Shutting down timeout executor."); - timeoutExecutor.shutdownNow(); - - log.log(Level.FINE, "Awaiting termination of already running timeout tasks."); - if (! timeoutExecutor.awaitTermination(300, TimeUnit.SECONDS)) { - log.severe("Did not manage to shut down the executors within 300 secs, system stuck?"); - throw new RuntimeException("Did not manage to shut down retry threads. Please report problem."); - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java deleted file mode 100644 index 1eebe593062..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationStats.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.communication.ClusterConnection; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.List; - -public class OperationStats { - - private static ObjectMapper jsonMapper = createMapper(); - - private final String sessionParamsAsXmlString; - private List<ClusterConnection> clusters; - private IncompleteResultsThrottler throttler; - - public OperationStats( - SessionParams sessionParams, - List<ClusterConnection> clusters, - IncompleteResultsThrottler throttler) { - this.sessionParamsAsXmlString = generateSessionParamsAsXmlString(sessionParams); - this.clusters = clusters; - this.throttler = throttler; - } - - private static ObjectMapper createMapper() { - ObjectMapper mapper = new ObjectMapper(); - mapper.registerModule(new Jdk8Module()); - mapper.registerModule(new JavaTimeModule()); - return mapper; - } - - private String generateSessionParamsAsXmlString(final SessionParams sessionParams) { - StringWriter stringWriter = new StringWriter(); - try { - JsonGenerator jsonGenerator = jsonMapper.createGenerator(stringWriter); - jsonMapper.writeValue(jsonGenerator, sessionParams); // TODO SessionParams should not be blindly serialized. This may serialize objects that are not really serializable. - return stringWriter.toString(); - } catch (IOException e) { - return e.getMessage(); - } - } - - public String getStatsAsJson() { - try { - StringWriter stringWriter = new StringWriter(); - JsonGenerator jsonGenerator = jsonMapper.createGenerator(stringWriter); - jsonGenerator.writeStartObject(); - jsonGenerator.writeArrayFieldStart("clusters"); - for (ClusterConnection cluster : clusters) { - jsonGenerator.writeStartObject(); - jsonGenerator.writeNumberField("clusterid", cluster.getClusterId()); - jsonGenerator.writeFieldName("stats"); - jsonGenerator.writeRawValue(cluster.getStatsAsJSon()); - jsonGenerator.writeEndObject(); - } - jsonGenerator.writeEndArray(); - jsonGenerator.writeFieldName("sessionParams"); - jsonGenerator.writeRawValue(sessionParamsAsXmlString); - jsonGenerator.writeFieldName("throttleDebugMessage"); - jsonGenerator.writeRawValue("\"" + throttler.getDebugMessage() + "\""); - jsonGenerator.writeEndObject(); - jsonGenerator.close(); - return stringWriter.toString(); - } catch (IOException e) { - return "{ \"Error\" : \""+ e.getMessage() + "\"}"; - } - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/package-info.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/package-info.java deleted file mode 100644 index 13d4e768daf..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/package-info.java +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java deleted file mode 100644 index 958d3793875..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * Programmatic API for feeding to Vespa clusters independently of the - * cluster configuration. - * - * NOTE: This is a PUBLIC API, but not annotated as such because this is not a bundle and - * we don't want to introduce Vespa dependencies. - */ -package com.yahoo.vespa.http.client; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java deleted file mode 100644 index 7ccdf3ebd43..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.google.common.base.Splitter; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import io.airlift.command.Command; -import io.airlift.command.HelpOption; -import io.airlift.command.Option; -import io.airlift.command.SingleCommand; -import org.apache.http.Header; -import org.apache.http.ParseException; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.message.BasicLineParser; - -import javax.inject.Inject; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -/** - * Commandline interface for the binary. - * - * @author dybis - */ -@Command(name = "vespa-http-client", - description = "This is a tool for feeding xml or json data to a Vespa application.") -public class CommandLineArguments { - - /** - * Creates a CommandLineArguments instance and populates it with data. - * - * @param args array of arguments. - * @return null on failure or if help option is set to true. - */ - static CommandLineArguments build(String[] args) { - CommandLineArguments cmdArgs; - try { - cmdArgs = SingleCommand.singleCommand(CommandLineArguments.class).parse(args); - } catch (Exception e) { - System.err.println(e.getMessage()); - System.err.println("Use --help to show usage.\n"); - return null; - } - if (cmdArgs.helpOption.showHelpIfRequested()) { - return null; - } - if (cmdArgs.endpointArg != null) { - if (cmdArgs.hostArg != null) { - System.err.println("Cannot set both '--host' and '--endpoint' "); - return null; - } - try { - URL url = new URL(cmdArgs.endpointArg); - } catch (MalformedURLException e) { - e.printStackTrace(System.err); - return null; - } - } else { - if (cmdArgs.hostArg == null) { - System.err.println("'--host' or '--endpoint' not set."); - return null; - } - } - if (cmdArgs.priorityArg != null && ! checkPriorityFlag(cmdArgs.priorityArg)) { - return null; - } - - for (String header : cmdArgs.headers) { - try { - cmdArgs.parsedHeaders.add(BasicLineParser.parseHeader(header, null)); - } catch (ParseException e) { - System.err.printf("Invalid header: '%s' (%s)%n", header, e.getMessage()); - return null; - } - } - - if (cmdArgs.privateKeyPath == null && cmdArgs.certificatePath != null || - cmdArgs.privateKeyPath != null && cmdArgs.certificatePath == null) { - System.err.println("Both '--privateKey' and '--certificate' must be set"); - return null; - } - - return cmdArgs; - } - - private static boolean checkPriorityFlag(String priorityArg) { - switch (priorityArg) { - case "HIGHEST": - case "VERY_HIGH": - case "HIGH_1": - case "HIGH_2": - case "HIGH_3": - case "NORMAL_1": - case "NORMAL_2": - case "NORMAL_3": - case "NORMAL_4": - case "NORMAL_5": - case "NORMAL_6": - case "LOW_1": - case "LOW_2": - case "LOW_3": - case "VERY_LOW": - case "LOWEST": - return true; - default: - System.err.println("Not valid value for priority. Allowed values are HIGHEST, VERY_HIGH, HIGH_[1-3], " + - "NORMAL_[1-6], LOW_[1-3], VERY_LOW, and LOWEST."); - return false; - } - } - - // TODO Don't duplicate default values from ConnectionParams.Builder. Some defaults are already inconsistent. - - @Inject - private HelpOption helpOption; - - @Option(name = {"--useV3Protocol"}, description = "Use V3 protocol to gateway. This is the default protocol.") - private boolean enableV3Protocol = true; - - @Option(name = {"--file"}, - description = "The name of the input file to read.") - private String fileArg = null; - - @Option(name = {"--add-root-element-to-xml"}, - description = "Add <vespafeed> tag to XML document, makes it easier to feed raw data.") - private boolean addRootElementToXml = false; - - @Option(name = {"--route"}, - description = "(=default)The route to send the data to.") - private String routeArg = "default"; - - @Option(name = {"--endpoint"}, - description = "Vespa endpoint.") - private String endpointArg; - - @Option(name = {"--host"}, - description = "The host(s) for the gateway. If using several, use comma to separate them.") - private String hostArg; - - @Option(name = {"--port"}, - description = "The port for the host of the gateway.") - private int portArg = 4080; - - @Option(name = {"--timeout"}, - description = "(=180) The time (in seconds) allowed for sending operations.") - private long timeoutArg = 180; - - @Option(name = {"--useCompression"}, - description = "Use compression over network.") - private boolean useCompressionArg = false; - - @Option(name = {"--useDynamicThrottling"}, - description = "Try to maximize throughput by using dynamic throttling.") - private boolean useDynamicThrottlingArg = false; - - @Option(name = {"--maxpending"}, - description = "The maximum number of operations that are allowed " + - "to be pending at any given time.") - private int maxPendingOperationCountArg = 10000; - - @Option(name = {"-v", "--verbose"}, - description = "Enable verbose output of progress.") - private boolean verboseArg = false; - - @Option(name = {"--noretry"}, - description = "Turns off retries of recoverable failures..") - private boolean noRetryArg = false; - - @Option(name = {"--retrydelay"}, - description = "The time (in seconds) to wait between retries of a failed operation.") - private int retrydelayArg = 1; - - @Option(name = {"--trace"}, - description = "(=0 (=off)) The trace level of network traffic.") - private int traceArg = 0; - - @Option(name = {"--printTraceEveryXOperation"}, - description = "(=1) How often to to tracing.") - private int traceEveryXOperation = 1; - - @Option(name = {"--validate"}, - description = "Run validation tool on input files instead of feeding them.") - private boolean validateArg = false; - - @Option(name = {"--ignoreConditionNotMet"}, - description = "Ignore condition not met failures.") - private boolean ignoreConditionNotMet = false; - - @Option(name = {"--priority"}, - description = "Specify priority of sent messages, see documentation ") - private String priorityArg = null; - - @Option(name = {"--numPersistentConnectionsPerEndpoint"}, - description = "How many tcp connections to establish per endoint.)") - private int numPersistentConnectionsPerEndpoint = 4; - - @Option(name = {"--maxChunkSizeBytes"}, - description = "How much data to send to gateway in each message.") - private int maxChunkSizeBytes = 20 * 1024; - - @Option(name = {"--whenVerboseEnabledPrintMessageForEveryXDocuments"}, - description = "How often to print verbose message.)") - private int whenVerboseEnabledPrintMessageForEveryXDocuments = 1000; - - @Option(name = {"--useTls"}, - description = "Use TLS when connecting to endpoint") - private boolean useTls = false; - - @Option(name = {"--insecure", "--disable-hostname-verification"}, - description = "Skip hostname verification when using TLS") - private boolean insecure = false; - - @Option(name = {"--header"}, - description = "Add http header to every request. Header must have the format '<Name>: <Value>'. Use this parameter multiple times for multiple headers") - private List<String> headers = new ArrayList<>(); - - @Option(name = {"--vespaTls"}, - description = "BETA! Use Vespa TLS configuration from environment if available. Other HTTPS/TLS configuration will be ignored if this is set.") - private boolean useTlsConfigFromEnvironment = false; - - @Option(name = {"--connectionTimeToLive"}, - description = "Maximum time to live for persistent connections. Specified as integer, in seconds.") - private long connectionTimeToLive = 15; - - @Option(name = {"--certificate"}, - description = "Path to a file containing a PEM encoded x509 certificate") - private String certificatePath; - - @Option(name = {"--privateKey"}, - description = "Path to a file containing a PEM encoded private key") - private String privateKeyPath; - - @Option(name = "--caCertificates", - description = "Path to a file containing a PEM encoded CA certificates") - private String caCertificatesPath; - - private final List<Header> parsedHeaders = new ArrayList<>(); - - int getWhenVerboseEnabledPrintMessageForEveryXDocuments() { - return whenVerboseEnabledPrintMessageForEveryXDocuments; - } - - public String getFile() { return fileArg; }; - - public boolean getVerbose() { return verboseArg; } - - public boolean getIgnoreConditionNotMet() { return ignoreConditionNotMet; } - - public boolean getAddRootElementToXml() { return addRootElementToXml; } - - SessionParams createSessionParams(boolean useJson) { - int minThrottleValue = useDynamicThrottlingArg ? 10 : 0; - Path privateKeyPath = Optional.ofNullable(this.privateKeyPath).map(Paths::get).orElse(null); - Path certificatePath = Optional.ofNullable(this.certificatePath).map(Paths::get).orElse(null); - Path caCertificatesPath = Optional.ofNullable(this.caCertificatesPath).map(Paths::get).orElse(null); - ConnectionParams.Builder connectionParamsBuilder = new ConnectionParams.Builder(); - parsedHeaders.forEach(header -> connectionParamsBuilder.addHeader(header.getName(), header.getValue())); - SessionParams.Builder builder = new SessionParams.Builder() - .setFeedParams( - new FeedParams.Builder() - .setDataFormat(useJson - ? FeedParams.DataFormat.JSON_UTF8 - : FeedParams.DataFormat.XML_UTF8) - .setRoute(routeArg) - .setMaxInFlightRequests(maxPendingOperationCountArg) - .setClientTimeout(timeoutArg, TimeUnit.SECONDS) - .setServerTimeout(timeoutArg, TimeUnit.SECONDS) - .setLocalQueueTimeOut(timeoutArg * 1000) - .setPriority(priorityArg) - .setMaxChunkSizeBytes(maxChunkSizeBytes) - .build() - ) - .setConnectionParams( - connectionParamsBuilder - .setHostnameVerifier(insecure ? NoopHostnameVerifier.INSTANCE : - SSLConnectionSocketFactory.getDefaultHostnameVerifier()) - .setUseCompression(useCompressionArg) - .setMaxRetries(noRetryArg ? 0 : 100) - .setMinTimeBetweenRetries(retrydelayArg, TimeUnit.SECONDS) - .setDryRun(validateArg) - .setTraceLevel(traceArg) - .setTraceEveryXOperation(traceEveryXOperation) - .setPrintTraceToStdErr(traceArg > 0) - .setNumPersistentConnectionsPerEndpoint(numPersistentConnectionsPerEndpoint) - .setCertificateAndPrivateKey(privateKeyPath, certificatePath) - .setCaCertificates(caCertificatesPath) - .setUseTlsConfigFromEnvironment(useTlsConfigFromEnvironment) - .setConnectionTimeToLive(Duration.ofSeconds(connectionTimeToLive)) - .build() - ) - // Enable dynamic throttling. - .setThrottlerMinSize(minThrottleValue) - .setClientQueueSize(maxPendingOperationCountArg); - if (endpointArg != null) { - try { - builder.addCluster(new Cluster.Builder() - .addEndpoint(Endpoint.create(new URL(endpointArg))) - .build()); - } - catch (MalformedURLException e) {} // already checked when parsing arguments - } - else { - Iterable<String> hosts = Splitter.on(',').trimResults().split(hostArg); - for (String host : hosts) { - builder.addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(host, portArg, useTls)) - .build()); - } - } - return builder.build(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java deleted file mode 100644 index dd6fbc29e5f..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/FormatInputStream.java +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.format.DataFormatDetector; -import com.fasterxml.jackson.core.format.DataFormatMatcher; -import com.fasterxml.jackson.core.format.MatchStrength; -import com.fasterxml.jackson.dataformat.xml.XmlFactory; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.SequenceInputStream; -import java.util.Arrays; -import java.util.Collections; -import java.util.Optional; - -/** - * @author valerijf - */ -public class FormatInputStream { - - private InputStream inputStream; - private final Format format; - - /** - * Creates a single data input stream from either file or InputStream depending on which one is present. Preference - * for file if both present. Additionally also detects input data format of the result stream, throws - * IllegalArgumentException if unable to determine data format. - * - * @param stream InputStream of the data if present - * @param inputFile path to file to use as input - * @param addRootElementToXml to add vespafeed root element around the input data stream - * @throws IOException on errors - */ - public FormatInputStream(InputStream stream, Optional<String> inputFile, boolean addRootElementToXml) - throws IOException { - DataFormatDetector dataFormatDetector = new DataFormatDetector(new JsonFactory(), new XmlFactory()); - DataFormatMatcher formatMatcher; - - if (inputFile.isPresent()) { - try (FileInputStream fileInputStream = new FileInputStream(inputFile.get())) { - formatMatcher = dataFormatDetector.findFormat(fileInputStream); - } - inputStream = new FileInputStream(inputFile.get()); - - } else { - if (stream.available() == 0) - System.out.println("No data in stream yet and no file specified, waiting for data."); - - inputStream = stream.markSupported() ? stream : new BufferedInputStream(stream); - inputStream.mark(DataFormatDetector.DEFAULT_MAX_INPUT_LOOKAHEAD); - formatMatcher = dataFormatDetector.findFormat(inputStream); - inputStream.reset(); - } - - if (addRootElementToXml) { - inputStream = addVespafeedTag(inputStream); - format = Format.XML; - return; - } - - if (formatMatcher.getMatchStrength() == MatchStrength.INCONCLUSIVE - || formatMatcher.getMatchStrength() == MatchStrength.NO_MATCH) { - throw new IllegalArgumentException("Could not detect input format"); - } - - switch (formatMatcher.getMatchedFormatName().toLowerCase()) { - case "json": - format = Format.JSON; - break; - case "xml": - format = Format.XML; - break; - default: - throw new IllegalArgumentException("Unknown data format"); - } - } - - private static InputStream addVespafeedTag(InputStream inputStream) { - return new SequenceInputStream(Collections.enumeration(Arrays.asList( - new ByteArrayInputStream("<vespafeed>".getBytes()), inputStream, - new ByteArrayInputStream("</vespafeed>".getBytes()))) - ); - } - - public InputStream getInputStream() { - return inputStream; - } - - public Format getFormat() { - return format; - } - - public enum Format { - JSON, XML - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java deleted file mode 100644 index e3e90c8bbfc..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/Runner.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedClientFactory; -import com.yahoo.vespa.http.client.SimpleLoggerResultCallback; -import com.yahoo.vespa.http.client.core.JsonReader; -import com.yahoo.vespa.http.client.core.XmlFeedReader; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.time.Clock; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @author Einar M R Rosenvinge - * @author dybis - */ -public class Runner { - - /** - * Feed data from inputFile to session. - * - * @param feedClient where to send data to - * @param inputStream source of data - * @param isJson if input stream is of json formatted data - * @param numSent is updated while sending by this method - * @param verbose if true will print some information to stderr - * @return send time in ms, not including validating - */ - public static long send(FeedClient feedClient, - InputStream inputStream, - boolean isJson, - AtomicInteger numSent, - boolean verbose) { - Clock clock = Clock.systemUTC(); - if (verbose) - System.err.println("Now sending data."); - - long sendStartTime = clock.millis(); - if (isJson) { - JsonReader.read(inputStream, feedClient, numSent); - } else { - try { - XmlFeedReader.read(inputStream, feedClient, numSent); - } catch (Exception e) { - System.err.println("Stopped reading feed, got problems with XML: " + e.getMessage()); - } - } - - long sendTotalTime = clock.millis() - sendStartTime; - - if (verbose) - System.err.println("Waiting for all results, sent " + numSent.get() + " docs."); - - feedClient.close(); - if (verbose) - System.err.println("Session closed."); - return sendTotalTime; - } - - - public static void main(String[] args) throws IOException { - CommandLineArguments commandLineArgs = CommandLineArguments.build(args); - if (commandLineArgs == null) - System.exit(1); - - FormatInputStream formatInputStream = new FormatInputStream(System.in, - Optional.ofNullable(commandLineArgs.getFile()), - commandLineArgs.getAddRootElementToXml()); - - int intervalOfLogging = - commandLineArgs.getVerbose() - ? commandLineArgs.getWhenVerboseEnabledPrintMessageForEveryXDocuments() - : Integer.MAX_VALUE; - AtomicInteger numSent = new AtomicInteger(0); - SimpleLoggerResultCallback callback = new SimpleLoggerResultCallback(numSent, intervalOfLogging, commandLineArgs.getIgnoreConditionNotMet()); - - FeedClient feedClient = FeedClientFactory.create(commandLineArgs.createSessionParams(formatInputStream.getFormat()== FormatInputStream.Format.JSON), - callback); - - long sendTotalTimeMs = send(feedClient, - formatInputStream.getInputStream(), - formatInputStream.getFormat() == FormatInputStream.Format.JSON, - numSent, - commandLineArgs.getVerbose()); - - if (commandLineArgs.getVerbose()) { - System.err.println(feedClient.getStatsAsJson()); - double transferTimeSec = ((double) sendTotalTimeMs) / 1000.0; - if (transferTimeSec > 0) - System.err.printf("Docs/sec %.3f%n", numSent.get() / transferTimeSec); - - if (commandLineArgs.getFile() != null) { - double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0; - System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds."); - System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " + - "(not taking compression into account)"); - } - } - callback.printProgress(); - } - -} diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/package-info.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/package-info.java deleted file mode 100644 index 8be767d67cf..00000000000 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/package-info.java +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; diff --git a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java b/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java deleted file mode 100644 index 2ca2ea4d14a..00000000000 --- a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedClientFactory; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.Server; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.handlers.V3MockParsingRequestHandler; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Unit test that test documentation code. - * - * @author dybis - */ -public class ExampleUsageFeedClientTest { - - @Test - public void testExampleCode() { - Server serverA = - new Server(new V3MockParsingRequestHandler(200, V3MockParsingRequestHandler.Scenario.ALL_OK), 0); - Server serverB = - new Server(new V3MockParsingRequestHandler(200, V3MockParsingRequestHandler.Scenario.ALL_OK), 0); - - exampleCode("localhost", serverA.getPort(), "localhost", serverB.getPort()); - serverA.close(); - serverB.close(); - } - - private static CharSequence generateDocument(String docId) { - // Just a dummy example of an update document operation. - return "{\"update\": \""+ docId + "\"," - + " \"fields\": { \"actualMapStringToArrayOfInt\": {" - + " \"assign\": [" - + "{ \"key\": \"fooKey\", \"value\": [ 2,1, 3] }" - + "]}}}"; - } - - // Example usage of FeedClient - public static void exampleCode(String hostNameA, int portServerA, String hostNameB, int portServerB) { - boolean useSsl = false; - final SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameA, portServerA, useSsl)).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameB, portServerB, useSsl)).build()) - .setFeedParams(new FeedParams.Builder() - .setDataFormat(FeedParams.DataFormat.JSON_UTF8) - .build()) - .build(); - - AtomicInteger resultsReceived = new AtomicInteger(0); - AtomicInteger errorsReceived = new AtomicInteger(0); - - FeedClient feedClient = FeedClientFactory.create(sessionParams, new FeedClient.ResultCallback() { - @Override - public void onCompletion(String docId, Result documentResult) { - resultsReceived.incrementAndGet(); - if (! documentResult.getContext().equals(docId)) { - System.err.println("Context does not work as expected."); - errorsReceived.incrementAndGet(); - } - if (!documentResult.isSuccess()) { - System.err.println("Problems with docID " + docId + ":" + documentResult.toString()); - errorsReceived.incrementAndGet(); - } - } - }); - int sentCounter = 0; - List<String> docIds = Arrays.asList("1", "2", "3", "4"); - for (final String docId : docIds) { - CharSequence docData = generateDocument(docId); - feedClient.stream(docId, docData, docId); - sentCounter++; - } - feedClient.close(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java deleted file mode 100644 index ca956110a34..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.api.FeedClientImpl; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Tests for the API, using dryrun option to mock gateway. - * - * @author dybis - */ -public class FeedClientTest { - - private final static String DOCID = "doc_id"; - - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder() - .addEndpoint(Endpoint.create("localhost")) - .build()) - .setConnectionParams(new ConnectionParams.Builder() - .setDryRun(true) - .build()) - .build(); - final AtomicInteger resultsReceived = new AtomicInteger(0); - - FeedClient.ResultCallback resultCallback = (docId, documentResult) -> { - assertTrue(documentResult.isSuccess()); - assertEquals(DOCID, docId); - resultsReceived.incrementAndGet(); - }; - - FeedClient feedClient = new FeedClientImpl(sessionParams, resultCallback, FeedClientFactory.createTimeoutExecutor(), Clock.systemUTC()); - - @Test - public void testStreamAndClose() { - feedClient.stream(DOCID, "blob"); - feedClient.close(); - assertEquals(1, resultsReceived.get()); - } - - @Test - public void testGetStatsAsJson() throws Exception { - feedClient.stream(DOCID, "blob"); - while (resultsReceived.get() == 0) {Thread.sleep(3); } - String stats = feedClient.getStatsAsJson(); - assertTrue(stats.contains("\"dryRun\":true")); - feedClient.close(); - } - - @Test - public void testFeedJson() { - InputStream stream = new ByteArrayInputStream((String.format("[{\"remove\": \"%s\"}]", DOCID) - .getBytes(StandardCharsets.UTF_8))); - AtomicInteger docCounter = new AtomicInteger(0); - FeedClient.feedJson(stream, feedClient, docCounter); - assertEquals(1, docCounter.get()); - feedClient.close(); - assertEquals(1, resultsReceived.get()); - } - - @Test - public void testFeedXml() { - InputStream stream = new ByteArrayInputStream((String.format( - "<document documenttype=\"music\" documentid=\"%s\">\n</document>\n", DOCID) - .getBytes(StandardCharsets.UTF_8))); - AtomicInteger docCounter = new AtomicInteger(0); - FeedClient.feedXml(stream, feedClient, docCounter); - assertEquals(1, docCounter.get()); - feedClient.close(); - assertEquals(1, resultsReceived.get()); - } - -} - diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java deleted file mode 100644 index 72edba7adb3..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.time.Clock; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.temporal.TemporalAmount; - -/** - * A clock which initially has the time of its creation but can only be advanced by calling advance - * - * @author bratseth - */ -public class ManualClock extends Clock { - - private Instant currentTime = Instant.now(); - - public ManualClock() {} - - public ManualClock(String utcIsoTime) { - this(at(utcIsoTime)); - } - - public ManualClock(Instant currentTime) { - this.currentTime = currentTime; - } - - public void advance(TemporalAmount temporal) { - currentTime = currentTime.plus(temporal); - } - - public void setInstant(Instant time) { - currentTime = time; - } - - @Override - public Instant instant() { return currentTime; } - - @Override - public ZoneId getZone() { return null; } - - @Override - public Clock withZone(ZoneId zone) { return null; } - - @Override - public long millis() { return currentTime.toEpochMilli(); } - - public static Instant at(String utcIsoTime) { - return LocalDateTime.parse(utcIsoTime, DateTimeFormatter.ISO_DATE_TIME).atZone(ZoneOffset.UTC).toInstant(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java deleted file mode 100644 index 078606274d6..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.handler.AbstractHandler; - -/** - * @author Einar M R Rosenvinge - */ -public final class Server implements AutoCloseable { - - private final org.eclipse.jetty.server.Server server; - - public Server(AbstractHandler handler, int port) { - this.server = new org.eclipse.jetty.server.Server(port); - server.setHandler(handler); - try { - server.start(); - assert(server.isStarted()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void close() throws RuntimeException { - try { - server.stop(); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException("jetty server.stop() failed", e); - } - } - - public int getPort() { - return ((ServerConnector)server.getConnectors()[0]).getLocalPort(); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java deleted file mode 100644 index 5f4a0fdec9a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import org.junit.Test; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class SimpleLoggerResultCallbackTest { - @Test - public void testAverageCalculation() { - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false); - Instant now = Instant.now(); - logger.newSamplingPeriod(now); - Result result = mock(Result.class); - when(result.isSuccess()).thenReturn(true); - // 3 documents in 0.2 secs --> 15 docs/sec - logger.onCompletion("1", result); - logger.onCompletion("1", result); - logger.onCompletion("1", result); - double rate = logger.newSamplingPeriod(now.plusMillis(200)).rate; - assertEquals(rate, 15., 0.1 /* delta */); - } - - @Test - public void testAverageCalculationExteremeValues() { - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false); - Instant now = Instant.now(); - logger.newSamplingPeriod(now); - // 0 duration, 0 documents - double rate = logger.newSamplingPeriod(now).rate; - assertEquals(rate, 0, 0.1 /* delta */); - } - - @Test - public void testOutput() { - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false); - Instant now = Instant.now(); - logger.newSamplingPeriod(now); - Result result = mock(Result.class); - when(result.isSuccess()).thenReturn(true); - // 3 documents in 0.2 secs --> 15 docs/sec - logger.onCompletion("1", result); - logger.onCompletion("1", result); - logger.onCompletion("1", result); - double rate = logger.newSamplingPeriod(now.plusMillis(200)).rate; - assertEquals(rate, 15., 0.1 /* delta */); - } - - private void verifyPrintout(boolean ignoreConditionNotMet) { - ArrayList<String> outputList = new ArrayList<>(); - - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(30), 0, ignoreConditionNotMet) { - @Override - protected void println(String output) { - outputList.add(output); - } - @Override - protected DocumentRate newSamplingPeriod(Instant now) { - return new DocumentRate(19999999.2342342366664); - } - }; - // 2 success, 1 failure - Result result = mock(Result.class); - when(result.isSuccess()).thenReturn(true); - when(result.isSuccessOrConditionNotMet()).thenReturn(true); - logger.onCompletion("1", result); - logger.onCompletion("1", result); - when(result.isSuccess()).thenReturn(false); - when(result.isSuccessOrConditionNotMet()).thenReturn(false); - when(result.toString()).thenReturn("fooError"); - logger.onCompletion("1", result); - logger.printProgress(); - assertThat(outputList.toString(), - containsString("Result received: 3 (1 failed so far, 30 sent, success rate 19999999.23 docs/sec).")); - assertThat(outputList.toString(), containsString("Failure: fooError")); - } - - @Test - public void testPrintout() { - verifyPrintout(false); - verifyPrintout(true); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java deleted file mode 100644 index 53b2b236d50..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.vespa.http.client.SyncFeedClient.SyncOperation; -import com.yahoo.vespa.http.client.SyncFeedClient.SyncResult; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNull; - -/** - * Tests the sync wrapper to the feed client - * - * @author bratseth - */ -public class SyncFeedClientTest { - - @Test - public void testFeedJson() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder() - .addEndpoint(Endpoint.create("localhost")) - .build()) - .setConnectionParams(new ConnectionParams.Builder() - .setDryRun(true) - .build()) - .build(); - SyncFeedClient feedClient = new SyncFeedClient(sessionParams); - - assertFeedSuccessful(feedClient); - assertFeedSuccessful(feedClient); // ensure the client can be reused - feedClient.close(); - } - - private void assertFeedSuccessful(SyncFeedClient feedClient) { - List<SyncOperation> operations = new ArrayList<>(); - - operations.add(new SyncOperation("id::test::1", - "{" + - " \"put\": \"id::test::1\"," + - " \"fields\": {" + - " \"title\": \"Title 1\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::2", - "{" + - " \"put\": \"id::test::2\"," + - " \"fields\": {" + - " \"title\": \"Title 2\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::3", - "{" + - " \"put\": \"id::test::3\"," + - " \"fields\": {" + - " \"title\": \"Title 3\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::3", // Another operation for the same document - "{" + - " \"put\": \"id::test::3\"," + - " \"fields\": {" + - " \"title\": \"Title 4\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::4", - "{" + - " \"put\": \"id::test::4\"," + - " \"fields\": {" + - " \"title\": \"Title 4\"" + - " }" + - "}", "opId_4", null)); - operations.add(new SyncOperation("id::test::4", // Another operation for the same document - "{" + - " \"put\": \"id::test::4\"," + - " \"fields\": {" + - " \"title\": \"Title 44\"" + - " }" + - "}", "opId_44", null)); - - SyncResult result = feedClient.stream(operations); - - assertTrue(result.isSuccess()); - assertEquals(6, result.results().size()); - assertNull(result.exception()); - assertEquals("id::test::1", result.results().get(0).getDocumentId()); - assertEquals("id::test::2", result.results().get(1).getDocumentId()); - assertEquals("id::test::3", result.results().get(2).getDocumentId()); - assertEquals("id::test::3", result.results().get(3).getDocumentId()); - assertEquals("id::test::4", result.results().get(4).getDocumentId()); - assertEquals("id::test::4", result.results().get(5).getDocumentId()); - assertEquals("opId_4", result.results().get(4).getOperationId()); - assertEquals("opId_44", result.results().get(5).getOperationId()); - assertTrue(result.results().get(4).getDocumentDataAsCharSequence().toString().contains("\"Title 4\"")); - assertTrue(result.results().get(5).getDocumentDataAsCharSequence().toString().contains("\"Title 44\"")); - - result.results().forEach(r -> assertNotNull(r.getOperationId())); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java deleted file mode 100644 index 67b3fc7653d..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -/** -* @author Einar M R Rosenvinge -*/ -public class TestDocument { - private final String documentId; - private final byte[] contents; - - TestDocument(String documentId, byte[] contents) { - this.documentId = documentId; - this.contents = contents; - } - - public String getDocumentId() { - return documentId; - } - - public byte[] getContents() { - return contents; - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java deleted file mode 100644 index b27fbba3e96..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPInputStream; - -import static org.junit.Assert.assertNull; - -/** - * @author Einar M R Rosenvinge - */ -@SuppressWarnings("deprecation") -public class TestUtils { - - public static void writeDocuments(Session session, List<TestDocument> documents) throws IOException { - for (TestDocument document : documents) { - writeDocument(session, document); - } - } - - public static void writeDocument(Session session, TestDocument document) throws IOException { - OutputStream operation = session.stream(document.getDocumentId()); - operation.write(document.getContents()); - operation.close(); - } - - public static Map<String, Result> getResults(Session session, int num) throws InterruptedException { - Map<String, Result> results = new HashMap<>(); - for (int i = 0; i < num; i++) { - Result r = session.results().poll(120, TimeUnit.SECONDS); - if (r == null) { - String extraInfo = ""; - extraInfo = "stats=" + session.getStatsAsJson(); - throw new IllegalStateException("Did not receive result within timeout. (" + extraInfo + ") " + - "Results received: " + results.values()); - } - results.put(r.getDocumentId(), r); - } - assertNull(session.results().poll(100, TimeUnit.MILLISECONDS)); - return results; - } - - public static String zipStreamToString(InputStream inputStream) throws IOException { - GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream); - final StringBuilder rawContent = new StringBuilder(); - while (true) { - int x = gzipInputStream.read(); - if (x < 0) { - break; - } - rawContent.append((char) x); - } - return rawContent.toString(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java deleted file mode 100644 index f4c12974f6d..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class ClusterTest { - - @Test - public void testSimple() { - Cluster cluster = new Cluster.Builder().build(); - - assertThat(cluster.getEndpoints().size(), is(0)); - assertThat(cluster.getRoute(), nullValue()); - } - - @Test - public void testConfig() { - Cluster cluster = new Cluster.Builder() - .addEndpoint(Endpoint.create("a")) - .addEndpoint(Endpoint.create("b")) - .setRoute("blah") - .build(); - - assertThat(cluster.getEndpoints().size(), is(2)); - assertThat(cluster.getEndpoints().get(0).getHostname(), equalTo("a")); - assertThat(cluster.getEndpoints().get(1).getHostname(), equalTo("b")); - assertThat(cluster.getRoute(), equalTo("blah")); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java deleted file mode 100644 index 3fcc73e3cdc..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import javax.net.ssl.SSLContext; -import java.security.NoSuchAlgorithmException; -import java.util.Iterator; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNull.nullValue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class ConnectionParamsTest { - - @Test - public void testDefaults() { - ConnectionParams params = new ConnectionParams.Builder().build(); - - assertThat(params.getHeaders().isEmpty(), is(true)); - assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(1)); - assertThat(params.getSslContext(), nullValue()); - } - - @Test - public void testSetters() throws NoSuchAlgorithmException { - ConnectionParams params = new ConnectionParams.Builder() - .addHeader("Foo", "Bar") - .addHeader("Foo", "Baz") - .addHeader("Banana", "Apple") - .setNumPersistentConnectionsPerEndpoint(2) - .setSslContext(SSLContext.getDefault()) - .build(); - - assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(2)); - - assertThat(params.getHeaders().isEmpty(), is(false)); - assertThat(params.getHeaders().size(), is(3)); - //Iteration order seems stable, let's keep it like this for now - Iterator<Map.Entry<String, String>> headers = params.getHeaders().iterator(); - Map.Entry<String, String> header1 = headers.next(); - assertThat(header1.getKey(), equalTo("Foo")); - assertThat(header1.getValue(), equalTo("Bar")); - Map.Entry<String, String> header2 = headers.next(); - assertThat(header2.getKey(), equalTo("Foo")); - assertThat(header2.getValue(), equalTo("Baz")); - Map.Entry<String, String> header3 = headers.next(); - assertThat(header3.getKey(), equalTo("Banana")); - assertThat(header3.getValue(), equalTo("Apple")); - } - - @Test - public void header_providers_are_registered() { - ConnectionParams.HeaderProvider dummyProvider1 = () -> "fooValue"; - ConnectionParams.HeaderProvider dummyProvider2 = () -> "barValue"; - ConnectionParams params = new ConnectionParams.Builder() - .addDynamicHeader("foo", dummyProvider1) - .addDynamicHeader("bar", dummyProvider2) - .build(); - Map<String, ConnectionParams.HeaderProvider> headerProviders = params.getDynamicHeaders(); - assertEquals(2, headerProviders.size()); - assertEquals(dummyProvider1, headerProviders.get("foo")); - assertEquals(dummyProvider2, headerProviders.get("bar")); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java deleted file mode 100644 index 415a7746b3a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class EndpointTest { - - @Test - public void testBasic() { - Endpoint endpoint = Endpoint.create("foo"); - - assertThat(endpoint.getHostname(), equalTo("foo")); - assertThat(endpoint.getPort(), equalTo(4080)); - assertThat(endpoint.isUseSsl(), is(false)); - } - - @Test - public void testBasicWithHttpProtocolPrefix() { - Endpoint endpoint = Endpoint.create("http://foo"); - assertThat(endpoint.getHostname(), equalTo("foo")); - } - - @Test(expected = RuntimeException.class) - public void testBasicWithHttpsProtocolPrefix() { - Endpoint.create("https://foo"); - } - - @Test - public void testAdvanced() { - Endpoint endpoint = Endpoint.create("bar", 1234, true); - - assertThat(endpoint.getHostname(), equalTo("bar")); - assertThat(endpoint.getPort(), equalTo(1234)); - assertThat(endpoint.isUseSsl(), is(true)); - } - - @Test - public void testMethods() { - Endpoint a = Endpoint.create("a"); - Endpoint b = Endpoint.create("b"); - - assertThat(a, not(equalTo(b))); - assertThat(a.hashCode(), not(equalTo(b.hashCode()))); - - Endpoint a2 = Endpoint.create("a"); - - assertThat(a, equalTo(a2)); - assertThat(a.hashCode(), equalTo(a2.hashCode())); - - a.toString(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java deleted file mode 100644 index 82e0b4deac0..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class FeedParamsTest { - - @Test - public void testDefaults() { - FeedParams params = new FeedParams.Builder().build(); - - assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.JSON_UTF8)); - assertThat(params.getMaxChunkSizeBytes(), is(50 * 1024)); - assertThat(params.getRoute(), nullValue()); - assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(180L)); - assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(20L)); - } - - @Test - public void testConfig() { - FeedParams params = new FeedParams.Builder() - .setDataFormat(FeedParams.DataFormat.XML_UTF8) - .setMaxChunkSizeBytes(123) - .setRoute("abc") - .setClientTimeout(321, TimeUnit.SECONDS) - .build(); - - assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.XML_UTF8)); - assertThat(params.getMaxChunkSizeBytes(), is(123)); - assertThat(params.getRoute(), equalTo("abc")); - assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(180L)); - assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(321L)); - - params = new FeedParams.Builder() - .setServerTimeout(333L, TimeUnit.SECONDS) - .setClientTimeout(222L, TimeUnit.SECONDS) - .build(); - - assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(333L)); - assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(222L)); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java deleted file mode 100644 index 8f707f2426e..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import org.junit.Test; - -import java.nio.ByteBuffer; -import java.nio.ReadOnlyBufferException; -import java.time.Clock; - -import static org.junit.Assert.assertEquals; - -public class DocumentTest { - - @Test - public void simpleCaseOk() { - String docId = "doc id"; - String docContent = "foo"; - Document document = new Document(docId, docContent.getBytes(), null, Clock.systemUTC().instant()); - assertEquals(docId, document.getDocumentId()); - assertEquals(ByteBuffer.wrap(docContent.getBytes()), document.getData()); - assertEquals(docContent, document.getDataAsString().toString()); - // Make sure that data is not modified on retrieval. - assertEquals(docContent, document.getDataAsString().toString()); - assertEquals(ByteBuffer.wrap(docContent.getBytes()), document.getData()); - assertEquals(docId, document.getDocumentId()); - } - - @Test(expected = ReadOnlyBufferException.class) - public void notMutablePutTest() { - Document document = new Document("id", null, "data", null, Clock.systemUTC().instant()); - document.getData().put("a".getBytes()); - } - - @Test(expected = ReadOnlyBufferException.class) - public void notMutableCompactTest() { - Document document = new Document("id", null, "data", null, Clock.systemUTC().instant()); - document.getData().compact(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java deleted file mode 100644 index aa926578a94..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import static org.junit.Assert.*; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Functional tests for encoding Encoder, i.e. encoding scheme only producing - * ASCII and never containing white space or control characters. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class EncoderTestCase { - private final String basic = "abc123"; - private final String basic2 = "abc{20}123"; - private final String quotedIsLast = "abc{20}"; - private final String quotedIsLastDecoded = "abc "; - private final String basic2Decoded = "abc 123"; - private final String unterminated = "abc{33"; - private final String unterminated2 = "abc{"; - private final String emptyQuoted = "abc{}123"; - private final String outsideUnicode = "abc{7fffffff}"; - private final String noise = "abc{7fff{||\\ffff}"; - private final String fullAsciiEncoded = "{0}{1}{2}{3}{4}{5}{6}{7}{8}{9}" - + "{a}{b}{c}{d}{e}{f}{10}{11}{12}{13}{14}{15}{16}{17}" - + "{18}{19}{1a}{1b}{1c}{1d}{1e}{1f}{20}" - + "!\"#$%&'()*+,-./0123456789:;<=>?@" - + "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`" - + "abcdefghijklmnopqrstuvwxyz{7b}|{7d}~{7f}"; - private final int[] testCodepoints = { 0x0, '\n', ' ', 'a', '{', '}', 0x7f, 0x80, - 0x7ff, 0x800, 0xd7ff, 0xe000, 0xffff, 0x10000, 0x10ffff, 0x34, - 0x355, 0x2567, 0xfff, 0xe987, 0x100abc }; - private final String semiNastyEncoded = "{0}{a}{20}a{7b}{7d}{7f}{80}" - + "{7ff}{800}{d7ff}{e000}{ffff}{10000}{10ffff}4" - + "{355}{2567}{fff}{e987}{100abc}"; - private final String invalidUnicode = "abc\ud812"; - private final String invalidUnicodeEncoded = "abc{d812}"; - - StringBuilder s; - - @Before - public void setUp() { - s = new StringBuilder(); - } - - @After - public void tearDown() { - s = null; - } - - @Test - public final void testBasic() { - Encoder.encode(basic, s); - assertEquals(basic, s.toString()); - } - - @Test - public final void testBasic2() { - Encoder.encode(basic2Decoded, s); - assertEquals(basic2, s.toString()); - } - - @Test - public final void testEncodeAscii() { - Encoder.encode(fullAscii(), s); - assertEquals(fullAsciiEncoded, s.toString()); - } - - @Test - public final void testEncodeMixed() { - Encoder.encode(semiNasty(), s); - assertEquals(semiNastyEncoded, s.toString()); - } - - @Test - public final void testEncodeQuotedIsLast() { - Encoder.encode(quotedIsLastDecoded, s); - assertEquals(quotedIsLast, s.toString()); - } - - @Test - public final void testInvalidUnicode() { - Encoder.encode(invalidUnicode, s); - assertEquals(invalidUnicodeEncoded, s.toString()); - } - - - @Test - public final void testDecodeBasic() { - Encoder.decode(basic, s); - assertEquals(basic, s.toString()); - } - - @Test - public final void testDecodeBasic2() { - Encoder.decode(basic2, s); - assertEquals(basic2Decoded, s.toString()); - } - - @Test - public final void testDecodeAscii() { - Encoder.decode(fullAsciiEncoded, s); - assertEquals(fullAscii(), s.toString()); - } - - @Test - public final void testDecodeMixed() { - Encoder.decode(semiNastyEncoded, s); - assertEquals(semiNasty(), s.toString()); - } - - - - @Test - public final void testDecodeQuotedIsLast() { - Encoder.decode(quotedIsLast, s); - assertEquals(quotedIsLastDecoded, s.toString()); - } - - - @Test - public final void testDecodeUnterminated() { - try { - Encoder.decode(unterminated, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - @Test - public final void testDecodeUnterminated2() { - try { - Encoder.decode(unterminated2, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - - } - - @Test - public final void testEmptyQuoted() { - try { - Encoder.decode(emptyQuoted, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - @Test - public final void testOutsideUnicode() { - try { - Encoder.decode(outsideUnicode, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - - @Test - public final void testNoise() { - try { - Encoder.decode(noise, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - @Test - public final void testIllegalInputCharacter() { - try { - Encoder.decode("abc\u00e5", s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - - @Test - public final void testNoIllegalCharactersInOutputForAscii() { - Encoder.encode(fullAscii(), s); - checkNoNonAscii(s.toString()); - } - - @Test - public final void testNoIllegalCharactersInOutputForMixedInput() { - Encoder.encode(semiNasty(), s); - checkNoNonAscii(s.toString()); - } - - @Test - public final void testSymmetryAscii() { - StringBuilder forDecoding = new StringBuilder(); - Encoder.encode(fullAscii(), s); - Encoder.decode(s.toString(), forDecoding); - assertEquals(fullAscii(), forDecoding.toString()); - } - - @Test - public final void testSymmetryMixed() { - StringBuilder forDecoding = new StringBuilder(); - Encoder.encode(semiNasty(), s); - Encoder.decode(s.toString(), forDecoding); - assertEquals(semiNasty(), forDecoding.toString()); - } - - - private void checkNoNonAscii(String input) { - for (int i = 0; i < input.length(); ++i) { - char c = input.charAt(i); - if (c > '~' || c <= ' ') { - fail("Encoded data contained character ordinal " + Integer.toHexString(c)); - } - } - } - - private String fullAscii() { - StringBuilder s = new StringBuilder(); - for (int i = 0; i <= 0x7f; ++i) { - s.append((char) i); - } - return s.toString(); - } - - private String semiNasty() { - StringBuilder s = new StringBuilder(); - for (int i = 0; i < testCodepoints.length; ++i) { - s.append(Character.toChars(testCodepoints[i])); - } - return s.toString(); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java deleted file mode 100644 index 2e6efffdd2c..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedEndpointException; -import com.yahoo.vespa.http.client.ManualClock; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.communication.ClusterConnection; -import com.yahoo.vespa.http.client.core.communication.IOThread; -import com.yahoo.vespa.http.client.core.communication.IOThreadTest; -import com.yahoo.vespa.http.client.core.operationProcessor.IncompleteResultsThrottler; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.time.Instant; -import java.util.List; -import java.util.concurrent.ScheduledThreadPoolExecutor; - -import static org.junit.Assert.assertEquals; - -/** - * Helper for testing with an operation processor - * - * @author bratseth - */ -public class OperationProcessorTester { - - private final Endpoint endpoint; - private final int clusterId = 0; - private final ManualClock clock; - private final TestResultCallback resultCallback; - private final OperationProcessor operationProcessor; - - public OperationProcessorTester() { - endpoint = Endpoint.create("test-endpoint"); - SessionParams.Builder params = new SessionParams.Builder(); - Cluster.Builder clusterParams = new Cluster.Builder(); - clusterParams.addEndpoint(endpoint); - params.addCluster(clusterParams.build()); - ConnectionParams.Builder connectionParams = new ConnectionParams.Builder(); - connectionParams.setDryRun(true); - connectionParams.setRunThreads(false); - params.setConnectionParams(connectionParams.build()); - - clock = new ManualClock(Instant.ofEpochMilli(0)); - resultCallback = new TestResultCallback(); - operationProcessor = new OperationProcessor(new IncompleteResultsThrottler(1, 100, clock, new ThrottlePolicy()), - resultCallback, - params.build(), - new ScheduledThreadPoolExecutor(1), - clock); - } - - public ManualClock clock() { return clock; } - - /** Asserts that this has but a single IOThread and returns it */ - public IOThread getSingleIOThread() { - assertEquals(1, clusterConnections().size()); - assertEquals(1, clusterConnections().get(0).ioThreads().size()); - return clusterConnections().get(0).ioThreads().get(0); - } - - /** Do n iteration of work in all io threads of this */ - public void tick(int n) { - for (int i = 0; i < n; i++) - for (ClusterConnection cluster : operationProcessor.clusters()) - for (IOThread thread : cluster.ioThreads()) - thread.tick(); - } - - public void send(String documentId) { - operationProcessor.sendDocument(new Document(documentId, documentId, "data of " + documentId, null, clock.instant())); - } - - public int incomplete() { - return operationProcessor.getIncompleteResultQueueSize(); - } - - public int success() { - return resultCallback.successes; - } - - public List<ClusterConnection> clusterConnections() { - return operationProcessor.clusters(); - } - - public int failures() { - return resultCallback.failures; - } - - public int endpointExceptions() { - return resultCallback.endpointExceptions; - } - - public Result lastResult() { - return resultCallback.lastResult; - } - - private static class TestResultCallback implements FeedClient.ResultCallback { - - private int successes = 0; - private int failures = 0; - private int endpointExceptions = 0; - private Result lastResult; - - @Override - public void onCompletion(String docId, Result documentResult) { - this.lastResult = documentResult; - if (documentResult.isSuccess()) - successes++; - else - failures++; - } - - @Override - public void onEndpointException(FeedEndpointException exception) { - endpointExceptions++; - } - - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java deleted file mode 100644 index d5a4d09fb8f..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import org.junit.Test; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -public class ThrottlePolicyTest { - - private final ThrottlePolicy throttlePolicy = new ThrottlePolicy(); - // Default values for tests. - private int numOk = 1000; - private int prevOk = 1000; - private int prevMax = 100; - private int max = 100; - private boolean queued = true; - private double dynamicFactor = 0.1; - - @Test - public void samePerformanceShouldTuneDown() { - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(95)); - } - - @Test - public void improvedPerformanceSameSizeShouldTuneDown() { - numOk += 200; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(89)); - } - - @Test - public void improvedPerformanceSmallerSizeTuneDownFurther() { - numOk += 200; - max = 70; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(63)); - } - - @Test - public void improvedPerformanceLargerSizeIncrease() { - numOk += 200; - max = 130; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(143)); - dynamicFactor = 100; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(156)); - } - - @Test - public void improvedPerformanceLargerSizeButQueuedFalse() { - numOk += 200; - max = 130; - queued = false; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(128)); - } - - @Test - public void lowerPerformanceSameSizeShouldIncrease() { - numOk -= 200; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(110)); - } - - @Test - public void lowerPerformanceSmallerSizeShouldIncreaseSize() { - numOk -= 200; - max = 30; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(33)); - dynamicFactor = 100; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(36)); - } - - @Test - public void lowerPerformanceLargerSizeTuneDownFurther() { - numOk -= 200; - max = 130; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(116)); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java deleted file mode 100644 index e71cee0f4c2..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.yahoo.vespa.http.client.FeedClient; -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.xml.sax.SAXParseException; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.Is.is; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; - -public class XmlFeedReaderTest { - private final static String feedResource = "/vespacorpfeed-prod-sample.xml"; - - private final static String feedResource2 = "/xml-challenge.xml"; - private final static String feedResource3 = "/xml-challenge2.xml"; - private final static String feedResource4 = "/xml-challenge3.xml"; - - private static final XmlMapper xmlMapper = new XmlMapper(); - - private final String updateDocUpdate = - "<?xml version=\"1.0\"?>\n" + - "<vespafeed>\n" + - "<update documentid=\"id:banana:banana::complex\" documenttype=\"banana\">\n" + - " <add fieldpath=\"structarr\">\n" + - " <item>\n" + - " <bytearr>\n" + - " <item>30</item>\n" + - " <item>55</item>\n" + - " </bytearr>\n" + - " </item>\n" + - " </add>\n" + - "</update>\n" + - "</vespafeed>\n"; - - @Test - public void testReadUpdate() throws Exception { - InputStream stream = new ByteArrayInputStream(updateDocUpdate.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(1)); - } - - private final String updateDocRemove = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "\n" + - "<vespafeed>\n" + - " <remove documentid=\"id:music:music::http://music.yahoo.com/Bob0/BestOf\" />\n" + - " <remove documentid=\"id:music:music::http://music.yahoo.com/Bob9/BestOf\" />\n" + - "</vespafeed>"; - - @Test - public void testReadRemove() throws Exception { - InputStream stream = new ByteArrayInputStream(updateDocRemove.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(2)); - } - - private final String insertDocOperation = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"+ - "<vespafeed>\n"+ - "\n"+ - " <document type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/bobdylan/BestOf\">\n"+ - " <title>Best of Bob Dylan</title>\n"+ - " </document>\n"+ - "\n"+ - " <document type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/metallica/BestOf\">\n"+ - " <title>Best of Metallica</title>\n"+ - " </document>\n"+ - "</vespafeed>"; - - @Test - public void testInsert() throws Exception { - InputStream stream = new ByteArrayInputStream(insertDocOperation.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(2)); - } - - private final String badperation = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"+ - "<vespafeed>\n"+ - " <badtag type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/bobdylan/BestOf\">\n"+ - " <title>Best of Bob Dylan</title>\n"+ - " </badtag>\n"+ - "</vespafeed>"; - - @Test - public void testNonDocument() throws Exception { - InputStream stream = new ByteArrayInputStream(badperation.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(0)); - } - - @Test(expected=SAXParseException.class) - public void testGarbage() throws Exception { - InputStream stream = new ByteArrayInputStream("eehh".getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - } - - @Test - public void testEncoding() throws Exception { - InputStream stream = new ByteArrayInputStream("<?xml version=\"1.0\" encoding=\"utf8\"?><vespafeed><remove documentid=\"id:&\"/></vespafeed>" - .getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - - doAnswer(new Answer<Object>() { - public Object answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String docId = (String) args[0]; - CharSequence value = (CharSequence)args[1]; - assertThat(value.toString(), is("<remove documentid=\"id:&\"></remove>")); - assertThat(docId, is("id:&")); - return null; - } - }).when(feedClient).stream(anyString(), any()); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(1)); - } - - private final String characterDocs = "" + - "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + - "<!-- GENERATED VESPA-XML BY YSSTOXML -->\n" + - "<!-- USE ONLY FOR BATCH INDEXING -->\n" + - "<vespafeed>\n" + - " <document documenttype=\"simple\" documentid=\"id:test::&http://www.e.no/matprat\">\n" + - " <language><![CDATA[ja]]></language>\n" + - " <title><![CDATA[test document1]]></title>\n" + - " <description><![CDATA[Bjørnen' blåbær på øy nærheten.]]></description>\n" + - " <date>1091356845</date>\n" + - " <surl><![CDATA[http://www.eventyr.no/matprat]]></surl>\n" + - " </document>\n" + - "\n" + - "</vespafeed>\n"; - - @Test - public void testCharacterEndcoding() throws Exception { - InputStream stream = new ByteArrayInputStream(characterDocs.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - final AtomicBoolean success = new AtomicBoolean(false); - doAnswer(new Answer<Object>() { - public Object answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String docId = (String) args[0]; - CharSequence value = (CharSequence)args[1]; - assertThat(value.toString(), is( - "<document documenttype=\"simple\" documentid=\"id:test::&http://www.e.no/matprat\">\n" + - " <language><![CDATA[ja]]></language>\n" + - " <title><![CDATA[test document1]]></title>\n" + - " <description><![CDATA[Bjørnen' blåbær på øy nærheten.]]></description>\n" + - " <date>1091356845</date>\n" + - " <surl><![CDATA[http://www.eventyr.no/matprat]]></surl>\n" + - " </document>")); - success.set(true); - return null; - } - }).when(feedClient).stream(anyString(), any()); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(1)); - assert(success.get()); - } - - @Test - public void testRealData() throws Exception { - InputStream inputStream = XmlFeedReaderTest.class.getResourceAsStream(feedResource); - BufferedInputStream bis = new BufferedInputStream(inputStream); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - - XmlFeedReader.read(bis, feedClient, numSent); - assertThat(numSent.get(), is(6)); - } - - private static class XmlTestFeedClient implements FeedClient { - - public List<String> documentIds = new ArrayList<>(); - public List<CharSequence> datas = new ArrayList<>(); - public List<Object> contexts = new ArrayList<>(); - - @Override - public void stream(String documentId, CharSequence documentData) { - stream(documentId, documentData, null); - } - - @Override - public void stream(String documentId, String operationId, CharSequence documentData, Object context) { - documentIds.add(documentId); - datas.add(documentData); - contexts.add(context); - } - - - @Override - public void close() { } - - @Override - public String getStatsAsJson() { return null; } - } - - // Only for xml with single doc. - private void verifyNoTransformationOfXml(String filename) throws Exception { - InputStream inputStream = XmlFeedReaderTest.class.getResourceAsStream(filename); - BufferedInputStream bis = new BufferedInputStream(inputStream); - AtomicInteger numSent = new AtomicInteger(0); - XmlTestFeedClient feedClient = new XmlTestFeedClient(); - XmlFeedReader.read(bis, feedClient, numSent); - assertThat(numSent.get(), is(1)); - String document = feedClient.datas.get(0).toString(); - - InputStream inputStream2 = XmlFeedReaderTest.class.getResourceAsStream(filename); - String rawXML = new java.util.Scanner(inputStream2, "UTF-8").useDelimiter("\\A").next(); - - JsonNode decodedDocument = xmlMapper.readTree(document); - JsonNode rawDocuments = xmlMapper.readTree(rawXML); - assertThat(decodedDocument, is(rawDocuments.get("document"))); - } - - @Test public void testCData() throws Exception { - verifyNoTransformationOfXml(feedResource2); - } - - @Test public void testPCData() throws Exception { - verifyNoTransformationOfXml(feedResource3); - } - - @Test public void testAposData() throws Exception { - verifyNoTransformationOfXml(feedResource4); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java deleted file mode 100644 index 283c1169440..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.api; - -import org.junit.Test; - -import java.time.Instant; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - -/** - * @author dybis - */ -public class FeedClientImplTest { - - int sleepValueMillis = 1; - - @Test - public void testCloseWaitTimeOldTimestamp() { - assertThat(FeedClientImpl.waitForOperations(Instant.now().minusSeconds(1000), sleepValueMillis, 10), is(false)); - } - - @Test - public void testCloseWaitTimeOutInFutureStillOperations() { - assertThat(FeedClientImpl.waitForOperations(Instant.now(), sleepValueMillis, 2000), is(true)); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java deleted file mode 100644 index 2597ddddd88..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java +++ /dev/null @@ -1,355 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.TestUtils; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import org.apache.http.Header; -import org.apache.http.HeaderElement; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.ParseException; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicHeader; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.stubbing.Answer; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class ApacheGatewayConnectionTest { - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Test - public void testProtocolV3() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().build(); - List<Document> documents = new ArrayList<>(); - - String vespaDocContent = "Hello, I a JSON doc."; - String docId = "42"; - - // This is the fake server, takes header client ID and uses this as session Id. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header clientIdHeader = post.getFirstHeader(Headers.CLIENT_ID); - return httpResponse(clientIdHeader.getValue(), "3"); - }); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - documents.add(createDoc(docId, vespaDocContent, true)); - - apacheGatewayConnection.write(documents); - } - - @Test(expected=IllegalArgumentException.class) - public void testServerReturnsBadSessionInV3() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().build(); - - // This is the fake server, returns wrong session Id. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> httpResponse("Wrong Id from server", "3")); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - List<Document> documents = new ArrayList<>(); - apacheGatewayConnection.write(documents); - } - - @Test - public void testJsonDocumentHeader() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build(); - List<Document> documents = new ArrayList<>(); - - String vespaDocContent ="Hello, I a JSON doc."; - String docId = "42"; - - AtomicInteger requestsReceived = new AtomicInteger(0); - - // This is the fake server, checks that DATA_FORMAT header is set properly. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header header = post.getFirstHeader(Headers.DATA_FORMAT); - if (requestsReceived.incrementAndGet() == 1) { - // This is handshake, it is not json. - assert (header == null); - return httpResponse("clientId", "3"); - } - assertNotNull(header); - assertEquals(FeedParams.DataFormat.JSON_UTF8.name(), header.getValue()); - // Test is done. - return httpResponse("clientId", "3"); - }); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - documents.add(createDoc(docId, vespaDocContent, true)); - - apacheGatewayConnection.write(documents); - } - - @Test - public void testZipAndCreateEntity() throws IOException { - String testString = "Hello world"; - InputStream stream = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8)); - // Send in test data to method. - InputStreamEntity inputStreamEntity = ApacheGatewayConnection.zipAndCreateEntity(stream); - // Verify zipped data by comparing unzipped data with test data. - String rawContent = TestUtils.zipStreamToString(inputStreamEntity.getContent()); - assertEquals(testString, rawContent); - } - - /** - * Mocks the HttpClient, and verifies that the compressed data is sent. - */ - @Test - public void testCompressedWriteOperations() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.XML_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build(); - List<Document> documents = new ArrayList<>(); - - String vespaDocContent ="Hello, I am the document data."; - String docId = "42"; - - Document doc = createDoc(docId, vespaDocContent, false); - - // When sending data on http client, check if it is compressed. If compressed, unzip, check result, - // and count down latch. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header header = post.getFirstHeader("Content-Encoding"); - if (header != null && header.getValue().equals("gzip")) { - final String rawContent = TestUtils.zipStreamToString(post.getEntity().getContent()); - final String vespaHeaderText = "<vespafeed>\n"; - final String vespaFooterText = "</vespafeed>\n"; - - assertEquals(doc.getOperationId() + " 38\n" + vespaHeaderText + vespaDocContent + "\n" + vespaFooterText, - rawContent); - } - return httpResponse("clientId", "3"); - }); - - StatusLine statusLineMock = mock(StatusLine.class); - when(statusLineMock.getStatusCode()).thenReturn(200); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - documents.add(doc); - - apacheGatewayConnection.write(documents); - } - - @Test - public void dynamic_headers_are_added_to_the_response() throws IOException, ServerResponseException { - ConnectionParams.HeaderProvider headerProvider = mock(ConnectionParams.HeaderProvider.class); - when(headerProvider.getHeaderValue()) - .thenReturn("v1") - .thenReturn("v2") - .thenReturn("v3"); - - ConnectionParams connectionParams = new ConnectionParams.Builder() - .addDynamicHeader("foo", headerProvider) - .build(); - - AtomicInteger counter = new AtomicInteger(1); - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header[] fooHeader = post.getHeaders("foo"); - assertEquals(1, fooHeader.length); - assertEquals("foo", fooHeader[0].getName()); - assertEquals("v" + counter.getAndIncrement(), fooHeader[0].getValue()); - return httpResponse("clientId", "3"); - - }); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - Endpoint.create("localhost", 666, false), - new FeedParams.Builder().build(), - "", - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - List<Document> documents = new ArrayList<>(); - documents.add(createDoc("42", "content", true)); - apacheGatewayConnection.write(documents); - apacheGatewayConnection.write(documents); - - verify(headerProvider, times(3)).getHeaderValue(); // 1x connect(), 2x writeOperations() - } - - @Test - public void detailed_error_message_is_extracted_from_error_responses_with_json() throws IOException, ServerResponseException { - String reasonPhrase = "Unauthorized"; - String errorMessage = "Invalid credentials"; - expectedException.expect(ServerResponseException.class); - expectedException.expectMessage(reasonPhrase + " - " + errorMessage); - - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> createErrorHttpResponse(401, reasonPhrase, errorMessage)); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - Endpoint.create("localhost", 666, false), - new FeedParams.Builder().build(), - "", - new ConnectionParams.Builder().build(), - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - apacheGatewayConnection.write(Collections.singletonList(createDoc("42", "content", true))); - } - - private static ApacheGatewayConnection.HttpClientFactory mockHttpClientFactory(HttpExecuteMock httpExecuteMock) throws IOException { - ApacheGatewayConnection.HttpClientFactory mockFactory = - mock(ApacheGatewayConnection.HttpClientFactory.class); - CloseableHttpClient httpClientMock = mock(CloseableHttpClient.class); - when(mockFactory.createClient()).thenReturn(httpClientMock); - when(httpClientMock.execute(any())).thenAnswer((Answer) invocation -> { - Object[] args = invocation.getArguments(); - HttpPost post = (HttpPost) args[0]; - return httpExecuteMock.execute(post); - }); - return mockFactory; - } - - @FunctionalInterface private interface HttpExecuteMock { - HttpResponse execute(HttpPost httpPost) throws IOException; - } - - private Document createDoc(String docId, String content, boolean useJson) { - return new Document(docId, content.getBytes(), null, Clock.systemUTC().instant()); - } - - private void addMockedHeader(HttpResponse httpResponseMock, String name, String value, HeaderElement[] elements) { - Header header = new Header() { - @Override - public String getName() { - return name; - } - @Override - public String getValue() { - return value; - } - @Override - public HeaderElement[] getElements() throws ParseException { - return elements; - } - }; - when(httpResponseMock.getFirstHeader(name)).thenReturn(header); - } - - private HttpResponse httpResponse(String sessionIdInResult, String version) throws IOException { - CloseableHttpResponse httpResponseMock = mock(CloseableHttpResponse.class); - - StatusLine statusLineMock = mock(StatusLine.class); - when(httpResponseMock.getStatusLine()).thenReturn(statusLineMock); - when(statusLineMock.getStatusCode()).thenReturn(200); - - addMockedHeader(httpResponseMock, Headers.SESSION_ID, sessionIdInResult, null); - addMockedHeader(httpResponseMock, Headers.VERSION, version, null); - HeaderElement[] headerElements = new HeaderElement[1]; - headerElements[0] = mock(HeaderElement.class); - - final HttpEntity httpEntityMock = mock(HttpEntity.class); - when(httpResponseMock.getEntity()).thenReturn(httpEntityMock); - - final InputStream inputs = new ByteArrayInputStream("fake response data".getBytes()); - - when(httpEntityMock.getContent()).thenReturn(inputs); - return httpResponseMock; - } - - private static HttpResponse createErrorHttpResponse(int statusCode, String reasonPhrase, String message) throws IOException { - CloseableHttpResponse response = mock(CloseableHttpResponse.class); - - StatusLine statusLine = mock(StatusLine.class); - when(statusLine.getStatusCode()).thenReturn(statusCode); - when(statusLine.getReasonPhrase()).thenReturn(reasonPhrase); - when(response.getStatusLine()).thenReturn(statusLine); - - HttpEntity httpEntity = mock(HttpEntity.class); - when(httpEntity.getContentType()).thenReturn(new BasicHeader("Content-Type", "application/json")); - String json = String.format("{\"message\": \"%s\"}", message); - when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream(json.getBytes())); - when(response.getEntity()).thenReturn(httpEntity); - return response; - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java deleted file mode 100644 index 9845768280e..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.communication.ByteBufferInputStream; -import org.junit.Test; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 - */ -public class ByteBufferInputStreamTest { - - private static ByteBuffer[] getAbcde() { - ByteBuffer[] buffers = new ByteBuffer[5]; - buffers[0] = ByteBuffer.wrap("a".getBytes(StandardCharsets.UTF_8)); - buffers[1] = ByteBuffer.wrap("b".getBytes(StandardCharsets.UTF_8)); - buffers[2] = ByteBuffer.wrap("c".getBytes(StandardCharsets.UTF_8)); - buffers[3] = ByteBuffer.wrap("d".getBytes(StandardCharsets.UTF_8)); - buffers[4] = ByteBuffer.wrap("e".getBytes(StandardCharsets.UTF_8)); - return buffers; - } - - @Test - public void requireThatExhaustedBufferWorks() throws IOException { - ByteBuffer[] buffers = getAbcde(); - buffers[2].get(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - - byte[] out = new byte[100]; - int pos = 0; - - final int GUARD = 1000; - int i; - for (i = 0; i < GUARD; i++) { - int r = in.read(); - if (r == -1) { - break; - } - out[pos] = (byte) (0xFF & r); - ++pos; - } - assertTrue(i < GUARD); - assertThat(pos, is(4)); - - String outString = new String(out, 0, pos, StandardCharsets.UTF_8); - assertThat(outString, equalTo("abde")); - - } - - @Test - public void requireThatBulkReadWorks() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - - byte[] out = new byte[100]; - int pos = 0; - - final int GUARD = 1000; - int i; - for (i = 0; i < GUARD; i++) { - int numReadNow; - if (i == 0) { - numReadNow = in.read(out); - } else { - numReadNow = in.read(out, pos, (out.length - pos)); - } - if (numReadNow == -1) { - break; - } - pos += numReadNow; - } - assertTrue(i < GUARD); - assertThat(pos, is(5)); - - String outString = new String(out, 0, pos, StandardCharsets.UTF_8); - assertThat(outString, equalTo("abcde")); - } - - @Test - public void requireThatSingleByteReadWorks() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - - byte[] out = new byte[100]; - int pos = 0; - - final int GUARD = 1000; - int i; - for (i = 0; i < GUARD; i++) { - int r = in.read(); - if (r == -1) { - break; - } - out[pos] = (byte) (0xFF & r); - ++pos; - } - assertTrue(i < GUARD); - assertThat(pos, is(5)); - - String outString = new String(out, 0, pos, StandardCharsets.UTF_8); - assertThat(outString, equalTo("abcde")); - } - - @Test - public void requireThatMarkIsNotSupported() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - assertThat(in.markSupported(), is(false)); - in.mark(0); //a no-op - } - - @Test(expected = IOException.class) - public void requireThatResetFails() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - in.reset(); - } - - @Test(expected = IOException.class) - public void requireThatSkipFails() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - in.skip(1L); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java deleted file mode 100644 index 227f4a5239c..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.Document; -import org.junit.Test; - -import java.time.Clock; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class CloseableQTestCase { - - @Test - public void requestThatPutIsInterruptedOnClose() throws InterruptedException { - Clock clock = Clock.systemUTC(); - DocumentQueue q = new DocumentQueue(1, clock); - q.put(new Document("id", null, "data", null, clock.instant()), false); - Thread t = new Thread(new Runnable() { - @Override - public void run() { - try { - Thread.sleep(3000); - } catch (InterruptedException e) { - } - q.close(); - q.clear(); - } - }); - t.start(); - try { - q.put(new Document("id2", null, "data2", null, Clock.systemUTC().instant()), false); - fail("This shouldn't have worked."); - } catch (IllegalStateException ise) { - // ok! - } - try { - t.join(); - } catch (InterruptedException e) { - } - } - - @Test - public void requireThatSelfIsUnbounded() throws InterruptedException { - DocumentQueue q = new DocumentQueue(1, Clock.systemUTC()); - q.put(new Document("1", null, "data", null, Clock.systemUTC().instant()), true); - q.put(new Document("2", null, "data", null, Clock.systemUTC().instant()), true); - q.put(new Document("3", null, "data", null, Clock.systemUTC().instant()), true); - assertEquals(3, q.size()); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java deleted file mode 100644 index 78ccfed3dbc..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; -import org.junit.Test; - -import java.time.Clock; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; - -/** - * @author Einar M R Rosenvinge - */ -public class EndpointResultQueueTest { - - @Test - public void testBasics() { - Endpoint endpoint = Endpoint.create("a"); - - GatewayConnection connection = new DryRunGatewayConnection(endpoint, Clock.systemUTC()); - OperationProcessor mockAggregator = mock(OperationProcessor.class); - final AtomicInteger resultCount = new AtomicInteger(0); - - doAnswer(invocationOnMock -> { - resultCount.getAndIncrement(); - return null; - }).when(mockAggregator).resultReceived(any(), eq(0)); - - EndpointResultQueue q = new EndpointResultQueue( - mockAggregator, endpoint, 0, new ScheduledThreadPoolExecutor(1), 100L * 1000L); - - q.operationSent("op1", connection); - assertThat(q.getPendingSize(), is(1)); - q.operationSent("op2", connection); - assertThat(q.getPendingSize(), is(2)); - q.operationSent("op3", connection); - assertThat(q.getPendingSize(), is(3)); - q.resultReceived(new EndpointResult("op1", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(2)); - q.resultReceived(new EndpointResult("op2", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(1)); - q.resultReceived(new EndpointResult("op3", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(0)); - q.resultReceived(new EndpointResult("op1", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(0)); - q.resultReceived(new EndpointResult("abc", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(0)); - - assertThat(resultCount.get(), is(5)); - - q.operationSent("op4", connection); - assertThat(q.getPendingSize(), is(1)); - q.operationSent("op5", connection); - assertThat(q.getPendingSize(), is(2)); - - q.failPending(new RuntimeException()); - - assertThat(resultCount.get(), is(7)); - } - - - @Test - public void testTimeout() throws InterruptedException { - Endpoint endpoint = Endpoint.create("a"); - OperationProcessor mockAggregator = mock(OperationProcessor.class); - CountDownLatch latch = new CountDownLatch(1); - doAnswer(invocationOnMock -> { - latch.countDown(); - return null; - }).when(mockAggregator).resultReceived(any(), eq(0)); - EndpointResultQueue q = new EndpointResultQueue( - mockAggregator, endpoint, 0, new ScheduledThreadPoolExecutor(1), 100L); - q.operationSent("1234", new DryRunGatewayConnection(endpoint, Clock.systemUTC())); - assert(latch.await(120, TimeUnit.SECONDS)); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java deleted file mode 100644 index 780947b525a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import org.junit.Before; -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - - -public class GatewayThrottlerTest { - - GatewayThrottler gatewayThrottler; - long lastSleepValue = 0; - - @Before - public void before() { - gatewayThrottler = new GatewayThrottler(900) { - @Override - protected void sleepMs(long sleepTime) { - lastSleepValue = sleepTime; - } - }; - } - - @Test - public void noSleepOnNormalCase() { - gatewayThrottler.handleCall(0); - gatewayThrottler.handleCall(0); - assertThat(lastSleepValue, is(0L)); - } - - @Test - public void increastingSleepTime() { - gatewayThrottler.handleCall(1); - long sleepTime1 = lastSleepValue; - gatewayThrottler.handleCall(1); - long sleepTime2 = lastSleepValue; - assertTrue(sleepTime1 > 0); - assertTrue(sleepTime2 > sleepTime1); - int x; - // Check for max value of sleep time. - for (x = 0 ; x < 10000; x++) { - long prevSleepTime = lastSleepValue; - gatewayThrottler.handleCall(1); - if (prevSleepTime == lastSleepValue) break; - } - assertTrue(x < 5000); - // Check that it goes down back to zero when no errors. - for (x = 0 ; x < 10000; x++) { - gatewayThrottler.handleCall(0); - if (lastSleepValue == 0) break; - } - assertTrue(x < 5000); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java deleted file mode 100644 index 9984e43374a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.OperationProcessorTester; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import org.junit.Test; - -import java.io.IOException; -import java.time.Duration; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; - -/** - * TODO: Migrate IOThreadTests here. - * - * @author bratseth - */ -public class IOThreadTest { - - @Test - public void testSuccessfulWriting() { - OperationProcessorTester tester = new OperationProcessorTester(); - assertEquals(0, tester.incomplete()); - assertEquals(0, tester.success()); - assertEquals(0, tester.failures()); - tester.send("doc1"); - tester.send("doc2"); - tester.send("doc3"); - assertEquals(3, tester.incomplete()); - assertEquals(0, tester.success()); - assertEquals(0, tester.failures()); - tester.tick(1); // connect - assertEquals(3, tester.incomplete()); - tester.tick(1); // sync - assertEquals(3, tester.incomplete()); - tester.tick(1); // process queue - assertEquals(0, tester.incomplete()); - assertEquals(3, tester.success()); - assertEquals(0, tester.failures()); - } - - @Test - public void testFatalExceptionOnHandshake() { - OperationProcessorTester tester = new OperationProcessorTester(); - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - firstConnection.throwOnHandshake(new ServerResponseException(403, "Not authorized")); - - tester.send("doc1"); - tester.send("doc2"); - tester.send("doc3"); - tester.tick(3); - assertEquals(0, tester.incomplete()); - assertEquals(0, ioThread.resultQueue().getPendingSize()); - assertEquals(0, tester.success()); - assertEquals(3, tester.failures()); - } - - @Test - public void testExceptionOnHandshake() { - OperationProcessorTester tester = new OperationProcessorTester(); - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - firstConnection.throwOnHandshake(new ServerResponseException(418, "I'm a teapot")); - - tester.send("doc1"); - tester.tick(3); - assertEquals(1, tester.incomplete()); - assertEquals(0, ioThread.resultQueue().getPendingSize()); - assertEquals(0, tester.success()); - assertEquals("Awaiting retry", 0, tester.failures()); - } - - @Test - public void testExceptionOnWrite() { - OperationProcessorTester tester = new OperationProcessorTester(); - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - firstConnection.throwOnWrite(new IOException("Test failure")); - - tester.send("doc1"); - tester.tick(3); - assertEquals(1, tester.incomplete()); - assertEquals(0, ioThread.resultQueue().getPendingSize()); - assertEquals(0, tester.success()); - assertEquals("Awaiting retry since write exceptions is a transient failure", - 0, tester.failures()); - } - - @Test - public void testPollingOldConnections() { - OperationProcessorTester tester = new OperationProcessorTester(); - tester.tick(3); - - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - assertEquals(0, ioThread.oldConnections().size()); - - firstConnection.hold(true); - tester.send("doc1"); - tester.tick(1); - - tester.clock().advance(Duration.ofSeconds(31)); // Default connection ttl is 30 - tester.tick(3); - - assertEquals(1, ioThread.oldConnections().size()); - assertEquals(firstConnection, ioThread.oldConnections().get(0)); - assertNotSame(firstConnection, ioThread.currentConnection()); - assertEquals(31, firstConnection.lastPollTime().toEpochMilli() / 1000); - - // Check old connection poll pattern (exponential backoff) - assertLastPollTimeWhenAdvancing(31, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - - tester.clock().advance(Duration.ofSeconds(200)); - tester.tick(1); - assertEquals("Old connection is eventually removed", 0, ioThread.oldConnections().size()); - } - - @Test - public void old_connections_are_closed_early_if_no_inflight_operations() { - OperationProcessorTester tester = new OperationProcessorTester(); - tester.tick(3); - - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - assertEquals(0, ioThread.oldConnections().size()); - - firstConnection.hold(true); // do not send result for doc1 in next http response - tester.send("doc1"); - tester.tick(1); - tester.clock().advance(Duration.ofSeconds(31)); // Default connection TTL + 1 - tester.tick(1); - assertEquals(1, ioThread.oldConnections().size()); - - firstConnection.hold(false); // send result for both doc1 and doc2 in next http response - tester.send("doc2"); - tester.tick(1); - assertEquals(1, ioThread.oldConnections().size()); - tester.clock().advance(Duration.ofSeconds(2)); - tester.tick(3); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertEquals(0, ioThread.oldConnections().size()); - } - - private void assertLastPollTimeWhenAdvancing(int lastPollTimeSeconds, - int advanceSeconds, - DryRunGatewayConnection connection, - OperationProcessorTester tester) { - tester.clock().advance(Duration.ofSeconds(advanceSeconds)); - tester.tick(1); - assertEquals(lastPollTimeSeconds, connection.lastPollTime().toEpochMilli() / 1000); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java deleted file mode 100644 index cd5da6b8589..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.core.operationProcessor.ConcurrentDocumentOperationBlocker; -import org.junit.Before; -import org.junit.Test; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -public class ConcurrentDocumentOperationBlockerTest { - - final ConcurrentDocumentOperationBlocker blocker = new ConcurrentDocumentOperationBlocker(); - final CountDownLatch latch = new CountDownLatch(1); - - @Before - public void setup() throws InterruptedException { - blocker.setMaxConcurrency(2); - blocker.startOperation(); - assertThat(blocker.availablePermits(), is(1)); - blocker.startOperation(); - } - - private void spawnThreadPushOperationThenCountDown() { - new Thread(() -> { - try { - blocker.startOperation(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - latch.countDown(); - }).start(); - } - - @Test - public void testBasics() throws InterruptedException { - spawnThreadPushOperationThenCountDown(); - assertFalse(latch.await(10, TimeUnit.MILLISECONDS)); - blocker.operationDone(); - assertTrue(latch.await(120, TimeUnit.SECONDS)); - } - - @Test - public void testResizeLarger() throws InterruptedException { - spawnThreadPushOperationThenCountDown(); - assertFalse(latch.await(10, TimeUnit.MILLISECONDS)); - blocker.setMaxConcurrency(3); - assertTrue(latch.await(120, TimeUnit.SECONDS)); - } - - @Test - public void testResizeSmaller() throws InterruptedException { - spawnThreadPushOperationThenCountDown(); - blocker.setMaxConcurrency(1); - blocker.operationDone(); - assertFalse(latch.await(10, TimeUnit.MILLISECONDS)); - blocker.operationDone(); - assertTrue(latch.await(120, TimeUnit.SECONDS)); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java deleted file mode 100644 index 82337cd4dcf..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.ManualClock; -import com.yahoo.vespa.http.client.core.ThrottlePolicy; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Random; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyDouble; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class IncompleteResultsThrottlerTest { - - @Test - public void simpleStaticQueueSizeTest() { - IncompleteResultsThrottler incompleteResultsThrottler = new IncompleteResultsThrottler(2, 2, null, null); - assertEquals(0, incompleteResultsThrottler.waitingThreads()); - incompleteResultsThrottler.operationStart(); - incompleteResultsThrottler.operationStart(); - assertEquals(2, incompleteResultsThrottler.waitingThreads()); - incompleteResultsThrottler.resultReady(true); - assertEquals(1, incompleteResultsThrottler.waitingThreads()); - incompleteResultsThrottler.resultReady(true); - assertEquals(0, incompleteResultsThrottler.waitingThreads()); - } - - /** - * Simulate running requests. - * @param clientCount number of parallel clients. - * @param breakPoint how many requests the server should handle in parallel before it gets slower. - * @param simulationTimeMs how many ms to simulate. - * @return median queue length. - */ - int getAverageQueue(int clientCount, int breakPoint, int simulationTimeMs) { - ManualClock clock = new ManualClock(Instant.ofEpochMilli(0)); - - ArrayList<IncompleteResultsThrottler> incompleteResultsThrottlers = new ArrayList<>(); - - MockServer mockServer = new MockServer(breakPoint); - for (int x = 0; x < clientCount; x++) { - IncompleteResultsThrottler incompleteResultsThrottler = - new IncompleteResultsThrottler(10, 50000, clock, new ThrottlePolicy()); - incompleteResultsThrottlers.add(incompleteResultsThrottler); - } - long sum = 0; - long samples = 0; - - for (long time = 0; time < simulationTimeMs; time++) { - // Fast forward, if we can. If all clients are blocked, we can move to the time when the server has a - // request that is finished. - boolean fastForward = true; - for (int x = 0; x < clientCount; x++) { - if (incompleteResultsThrottlers.get(x).availableCapacity() > 0 ) { - fastForward = false; - break; - } - } - if (fastForward) { - time = mockServer.nextRequestFinished(); - } - clock.setInstant(Instant.ofEpochMilli(time)); - mockServer.moveTime(clock.instant().toEpochMilli()); - for (int y = 0; y < clientCount; y++) { - // Fill up, but don't block as that would stop the simulation. - while (incompleteResultsThrottlers.get(y).availableCapacity() > 0) { - incompleteResultsThrottlers.get(y).operationStart(); - mockServer.newRequest(incompleteResultsThrottlers.get(y)); - } - } - // Don't take the first iterations into account as the system is eagerly learning. - if (time > 60*1000) { - sum += mockServer.messageDoneByTime.size(); - samples ++; - } - } - return (int)(sum/samples); - } - - private void testAndPrintVariousClientSizes(int breakPoint) { - final int sampleRuns = 6; - final int maxParallelClients = 4; - final int minParallelClients = 1; - final int simulationTimeMs = 400000; - System.out.print("\nBreakpoint is " + breakPoint + ", average queue on server:"); - int[][] resultQueuesAverage = new int[maxParallelClients][sampleRuns]; - for (int clientNo = minParallelClients; clientNo <= maxParallelClients; clientNo++) { - System.out.print("\nNow with " + clientNo + " parallel clients:"); - long sum = 0; - for (int x = 0; x < sampleRuns; x++) { - resultQueuesAverage[clientNo-minParallelClients][x] = getAverageQueue(1 + x, breakPoint, simulationTimeMs); - System.out.print(" " + resultQueuesAverage[clientNo-minParallelClients][x]); - sum += resultQueuesAverage[clientNo-minParallelClients][x]; - } - System.out.print(" average is " + sum/sampleRuns); - Arrays.sort(resultQueuesAverage[clientNo - minParallelClients]); - int median = resultQueuesAverage[clientNo - minParallelClients][sampleRuns/2]; - System.out.print(" median is " + median); - System.out.print(" min " + resultQueuesAverage[clientNo - minParallelClients][0]); - System.out.print(" max " + resultQueuesAverage[clientNo - minParallelClients][sampleRuns - 1]); - assertTrue(median < 2 * breakPoint + 200); - assertTrue(median > breakPoint / 10); - } - } - - @Test - public void testVariousBreakpoints() { - testAndPrintVariousClientSizes(200); - testAndPrintVariousClientSizes(1000); - } - - List<Thread> threads = new ArrayList<>(); - - private void postOperations(int count, final IncompleteResultsThrottler throttler) { - for (int i = 0; i < count; i++) { - Thread thread = new Thread(()->throttler.operationStart()); - thread.start(); - threads.add(thread); - } - } - - private void waitForThreads() throws InterruptedException { - while(!threads.isEmpty()) { - threads.remove(0).join(); - } - } - - private void postSuccesses(int count, final IncompleteResultsThrottler throttler) { - for (int i = 0; i < count; i++) { - throttler.resultReady(true); - } - } - - private void moveToNextCycle(final IncompleteResultsThrottler throttler, ManualClock clock) - throws InterruptedException { - waitForThreads(); - // Enter an adaption phase, we don't care about this phase. - clock.advance(Duration.ofMillis(throttler.phaseSizeMs)); - throttler.operationStart(); - throttler.resultReady(false); - // Now enter the real next phase. - clock.advance(Duration.ofMillis(throttler.phaseSizeMs)); - throttler.operationStart(); - throttler.resultReady(false); - } - - @Test - public void testInteractionWithPolicyByMockingPolicy() throws InterruptedException { - ManualClock clock = new ManualClock(Instant.ofEpochMilli(0)); - final int MAX_SIZE = 1000; - final int MORE_THAN_MAX_SIZE = MAX_SIZE + 20; - final int SIZE_AFTER_CYCLE_FIRST = 30; - final int SIZE_AFTER_CYCLE_SECOND = 5000; - ThrottlePolicy policy = mock(ThrottlePolicy.class); - IncompleteResultsThrottler incompleteResultsThrottler = - new IncompleteResultsThrottler(2, MAX_SIZE, clock, policy); - long bucketSizeMs = incompleteResultsThrottler.phaseSizeMs; - - // Cycle 1 - Algorithm has fixed value for max-in-flight: INITIAL_MAX_IN_FLIGHT_VALUE. - // We post a few operations, not all finishing in this cycle. We explicitly do not fill the window - // size to test the argument about any requests blocked. - assertEquals(IncompleteResultsThrottler.INITIAL_MAX_IN_FLIGHT_VALUE, - incompleteResultsThrottler.availableCapacity()); - postOperations(20, incompleteResultsThrottler); - postSuccesses(15, incompleteResultsThrottler); - moveToNextCycle(incompleteResultsThrottler, clock); - - - // Cycle 2 - Algorithm has fixed value also for second iteration: SECOND_MAX_IN_FLIGHT_VALUE. - // Test verifies that this value is used, and insert a value to be used for next phase SIZE_AFTER_CYCLE_FIRST. - assertEquals("5 slots already taken earlier", - IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE - 5, - incompleteResultsThrottler.availableCapacity()); - postSuccesses(5, incompleteResultsThrottler); - when(policy.calcNewMaxInFlight( - anyDouble(), // Max performance change - eq(5), //numOk - eq(15), // previousNumOk - eq(IncompleteResultsThrottler.INITIAL_MAX_IN_FLIGHT_VALUE), // previous size - eq(IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE), // current size - eq(false))) // is any request blocked, should be false since we only posted 20 docs. - .thenReturn(SIZE_AFTER_CYCLE_FIRST); - moveToNextCycle(incompleteResultsThrottler, clock); - - // Cycle 3 - Test that value set in previous phase is used. Now return a very large number. - // However, this number should be cropped by the system (tested in next cycle). - assertEquals(SIZE_AFTER_CYCLE_FIRST, incompleteResultsThrottler.availableCapacity()); - postOperations(MORE_THAN_MAX_SIZE, incompleteResultsThrottler); - postSuccesses(MORE_THAN_MAX_SIZE, incompleteResultsThrottler); - when(policy.calcNewMaxInFlight( - anyDouble(), // Max performance change - eq(MORE_THAN_MAX_SIZE), //numOk - eq(5), // previousNumOk - eq(IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE), // previous size - eq(SIZE_AFTER_CYCLE_FIRST),// current size - eq(true))) // is any request blocked, should be true since we posted MORE_THAN_MAX_SIZE docs. - .thenReturn(SIZE_AFTER_CYCLE_SECOND); - moveToNextCycle(incompleteResultsThrottler, clock); - - // Cycle 4 - Test that the large number from previous cycle is cropped and that max value is used instead. - assertEquals(MAX_SIZE, incompleteResultsThrottler.availableCapacity()); - } - - private long inversesU(int size, int sweetSpot) { - // Peak performance at sweetSPot. - int distance = Math.abs(sweetSpot - size); - return 1 + 20 * distance; - } - - /** - * A mock 'gateway' this is slower with more requests in-flight. It starts to become really much slower at - * 'breakPoint' number of parallel requests. - */ - class MockServer { - final LinkedList<Tuple2<Long, IncompleteResultsThrottler> > messageDoneByTime = new LinkedList<>(); - final int breakPoint; - final Random random = new Random(); - long time = 0; - - MockServer(int breakPoint) { - this.breakPoint = breakPoint; - } - - /** - * Figures out when next processed data will be ready. - * @return time in ms for next request to be finished. - */ - long nextRequestFinished() { - if (messageDoneByTime.isEmpty()) { - return Integer.MAX_VALUE; - } - return messageDoneByTime.peek().first; - } - - /** - * Advance simulation time and call finished on any requests. - * @param time to move to - */ - void moveTime(long time) { - this.time = time; - while (!messageDoneByTime.isEmpty() && messageDoneByTime.peek().first <= time) { - messageDoneByTime.pop().second.resultReady(true); - } - } - - /** - * New request. - * @param blocker do callback on blocker when request is done. - */ - void newRequest(IncompleteResultsThrottler blocker) { - long nextTime = (long)(20 + 0.1 * messageDoneByTime.size()); - - if (messageDoneByTime.size() > breakPoint) { - nextTime += (long) (40 + (random.nextDouble()) * 0.01 * messageDoneByTime.size()* messageDoneByTime.size()); - } - nextTime += time + random.nextInt()%4; - messageDoneByTime.push(new Tuple2<>(nextTime, blocker)); - } - } - - private static class Tuple2<T1, T2> { - - public final T1 first; - public final T2 second; - - public Tuple2(final T1 first, final T2 second) { - this.first = first; - this.second = second; - } - - @Override - public int hashCode() { throw new UnsupportedOperationException(); } - - @Override - public boolean equals(final Object obj) { throw new UnsupportedOperationException(); } - - @Override - public String toString() { - return "Tuple2(" + first + ", " + second + ")"; - } - - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java deleted file mode 100644 index 7799f6089db..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.EndpointResult; -import org.junit.Test; - -import java.time.Clock; -import java.util.ArrayDeque; -import java.util.Queue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author Einar M R Rosenvinge - */ -public class OperationProcessorTest { - - final Queue<Result> queue = new ArrayDeque<>(); - final Document doc1 = new Document("id:a:type::b", null, "data doc 1", null, Clock.systemUTC().instant()); - final Document doc1b = new Document("id:a:type::b", null, "data doc 1b", null, Clock.systemUTC().instant()); - final Document doc2 = new Document("id:a:type::b2", null, "data doc 2", null, Clock.systemUTC().instant()); - final Document doc3 = new Document("id:a:type::b3", null, "data doc 3", null, Clock.systemUTC().instant()); - - @Test - public void testBasic() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor q = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - - q.resultReceived(new EndpointResult("foo", new Result.Detail(null)), 0); - assertEquals(0, queue.size()); - - - q.sendDocument(doc1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("d"))), 3); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("e"))), 0); - assertEquals(1, queue.size()); - - //check a, b, c, d - Result aggregated = queue.poll(); - assertEquals("id:a:type::b", aggregated.getDocumentId()); - assertEquals(4, aggregated.getDetails().size()); - assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname()); - assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname()); - assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname()); - assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname()); - assertEquals("data doc 1", aggregated.getDocumentDataAsCharSequence().toString()); - - assertEquals(0, queue.size()); - - q.sendDocument(doc2); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("d"))), 3); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("e"))), 0); - assertEquals(1, queue.size()); - - // check a, b, c, d - aggregated = queue.poll(); - assertEquals("id:a:type::b2", aggregated.getDocumentId()); - assertEquals(4, aggregated.getDetails().size()); - assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname()); - assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname()); - assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname()); - assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname()); - assertEquals("data doc 2", aggregated.getDocumentDataAsCharSequence().toString()); - - assertEquals(0, queue.size()); - } - - @Test - public void testBlockingOfOperationsTwoEndpoints() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc1b); - - assertEquals(0, queue.size()); - // Only one operations should be in flight. - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(0, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertEquals(1, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(1, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertEquals(2, queue.size()); - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - // This should have no effect. - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertEquals(2, queue.size()); - } - - @Test - public void testBlockingOfOperationsToSameDocIdWithTwoOperations() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc1b); - - assertEquals(0, queue.size()); - // Only one operations should be in flight. - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(1, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(2, queue.size()); - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - assertFalse(operationProcessor.oldestIncompleteResultId().isPresent()); - // This should have no effect. - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(2, queue.size()); - } - - @Test - public void testBlockingOfOperationsToSameDocIdMany() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - Queue<Document> documentQueue = new ArrayDeque<>(); - for (int x = 0; x < 100; x++) { - Document document = new Document("id:a:type::b", null, String.valueOf(x), null, Clock.systemUTC().instant()); - operationProcessor.sendDocument(document); - documentQueue.add(document); - } - - for (int x = 0; x < 100; x++) { - assertEquals(x, queue.size()); - // Only one operations should be in flight. - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - Document document = documentQueue.poll(); - operationProcessor.resultReceived(new EndpointResult(document.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(x+1, queue.size()); - if (x < 99) { - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - } else { - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - } - } - } - - @Test - public void testMixOfBlockingAndNonBlocking() { - Endpoint endpoint = Endpoint.create("localhost"); - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(endpoint).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc1b); // Blocked - operationProcessor.sendDocument(doc2); - operationProcessor.sendDocument(doc3); - - assertEquals(0, queue.size()); - assertEquals(3, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - // This should have no effect since it should not be sent. - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(3, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - - operationProcessor.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(2, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - assertFalse(operationProcessor.oldestIncompleteResultId().isPresent()); - } - - @Test - public void assertThatDuplicateResultsFromOneClusterWorks() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor q = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - q.sendDocument(doc1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 0); - assertEquals(0, queue.size()); - } - - @Test - public void testMultipleDuplicateDocIds() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor q = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - q.sendDocument(doc1); - assertEquals(0, queue.size()); - q.sendDocument(doc2); - assertEquals(0, queue.size()); - q.sendDocument(doc3); - - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(2, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(2, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(2, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(3, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(3, queue.size()); - assertEquals("data doc 1", queue.remove().getDocumentDataAsCharSequence().toString()); - assertEquals("data doc 2", queue.remove().getDocumentDataAsCharSequence().toString()); - assertEquals("data doc 3", queue.remove().getDocumentDataAsCharSequence().toString()); - } - - @Test - public void testWaitBlocks() throws InterruptedException { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1, 1, null, null), - (docId, documentResult) -> {}, - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - - final CountDownLatch started = new CountDownLatch(1); - final CountDownLatch done = new CountDownLatch(1); - - Thread shouldWait = new Thread(()-> { - started.countDown(); - operationProcessor.sendDocument(doc2); - done.countDown(); - }); - shouldWait.start(); - started.await(); - // We want the test to pass fast so we only wait 40mS to see that it is blocking. This might lead to - // some false positives, but that is ok. - assertFalse(done.await(40, TimeUnit.MILLISECONDS)); - operationProcessor.resultReceived( - new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("d"))), 0); - assertTrue(done.await(120, TimeUnit.SECONDS)); - - } - - @Test - public void testSendsResponseToQueuedDocumentOnClose() throws InterruptedException { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - ScheduledThreadPoolExecutor executor = mock(ScheduledThreadPoolExecutor.class); - when(executor.awaitTermination(anyLong(), any())).thenReturn(true); - - CountDownLatch countDownLatch = new CountDownLatch(3); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(19, 19, null, null), - (docId, documentResult) -> { - countDownLatch.countDown(); - }, - sessionParams, executor, Clock.systemUTC()); - - // Will fail due to bogus host name, but will be retried. - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc2); - operationProcessor.sendDocument(doc3); - - // Will create fail results. - operationProcessor.close(); - countDownLatch.await(); - } - - @Test - public void unknownHostThrowsExceptionAtConstructionTime() { - try { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("unknown.invalid")).build()) - .build(); - ScheduledThreadPoolExecutor executor = mock(ScheduledThreadPoolExecutor.class); - - CountDownLatch countDownLatch = new CountDownLatch(3); - - new OperationProcessor( - new IncompleteResultsThrottler(19, 19, null, null), - (docId, documentResult) -> { - countDownLatch.countDown(); - }, - sessionParams, executor, Clock.systemUTC()); - - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("Unknown host: unknown.invalid:4080 ssl=false", e.getMessage()); - } - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java deleted file mode 100644 index bf4d4d0800b..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.handlers; - -import com.yahoo.vespa.http.client.core.Encoder; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.handler.AbstractHandler; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.27 - */ -public class V3MockParsingRequestHandler extends AbstractHandler { - private final int responseCode; - private volatile Scenario scenario; - private final BlockingQueue<CountDownLatch> delayedRequests = new LinkedBlockingQueue<>(); - private final AtomicBoolean delayedResponseShouldBlock = new AtomicBoolean(true); - public final AtomicBoolean badRequestScenarioShouldReturnBadRequest = new AtomicBoolean(false); - private final String name; - private static final AtomicInteger sessionIdGenerator = new AtomicInteger(0); - private AtomicInteger internalCounter = new AtomicInteger(0); - - public enum Scenario { - ALL_OK, RETURN_WRONG_SESSION_ID, - DISCONNECT_IMMEDIATELY, DONT_ACCEPT_VERSION, RETURN_UNEXPECTED_VERSION, - INTERNAL_SERVER_ERROR, COULD_NOT_FEED, MBUS_RETURNED_ERROR, - NEVER_RETURN_ANY_RESULTS, DELAYED_RESPONSE, BAD_REQUEST, SERVER_ERROR_TWICE_THEN_OK, - EXPECT_HIGHEST_PRIORITY_AND_TRACELEVEL_123, CONDITON_NOT_MET - } - - public V3MockParsingRequestHandler() { - this("", HttpServletResponse.SC_OK, Scenario.ALL_OK); - } - - public V3MockParsingRequestHandler(String name) { - this(name, HttpServletResponse.SC_OK, Scenario.ALL_OK); - } - - public V3MockParsingRequestHandler(int responseCode) { - this("", responseCode, Scenario.ALL_OK); - } - - public V3MockParsingRequestHandler(int responseCode, Scenario scenario) { - this("", responseCode, scenario); - } - - public V3MockParsingRequestHandler(String name, int responseCode, Scenario scenario) { - this.name = name; - this.responseCode = responseCode; - this.scenario = scenario; - } - - public void setScenario(Scenario scenario) { - this.scenario = scenario; - } - - @Override - public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - System.err.println("Server " + name + " got request from: " + request.getHeader(Headers.SESSION_ID)); - switch (scenario) { - case ALL_OK: - allOk(baseRequest, request, response); - break; - case RETURN_WRONG_SESSION_ID: - wrongSessionId(baseRequest, request, response); - break; - case DISCONNECT_IMMEDIATELY: - disconnect(baseRequest, response); - break; - case DONT_ACCEPT_VERSION: - dontAcceptVersion(baseRequest, request, response); - break; - case RETURN_UNEXPECTED_VERSION: - unexpectedVersion(baseRequest, request, response); - break; - case INTERNAL_SERVER_ERROR: - internalServerError(baseRequest, request, response); - break; - case COULD_NOT_FEED: - couldNotFeed(baseRequest, request, response); - break; - case MBUS_RETURNED_ERROR: - mbusReturnedError(baseRequest, request, response); - break; - case NEVER_RETURN_ANY_RESULTS: - neverReturnAnyResults(baseRequest, request, response); - break; - case DELAYED_RESPONSE: - delayedResponse(baseRequest, request, response); - break; - case BAD_REQUEST: - badRequest(baseRequest, request, response); - break; - case SERVER_ERROR_TWICE_THEN_OK: - int state = internalCounter.getAndIncrement(); - if (state >= 2) { - allOk(baseRequest, request, response); - } else { - couldNotFeed(baseRequest, request, response); - } - break; - case EXPECT_HIGHEST_PRIORITY_AND_TRACELEVEL_123: - checkIfSessionThenHighPriorityAndTraceLevel123(request); - allOk(baseRequest, request, response); - break; - case CONDITON_NOT_MET: - conditionNotMetRequest(baseRequest, request, response); - break; - default: - throw new IllegalArgumentException("Test scenario " + scenario + " not supported."); - } - } - - private void checkIfSessionThenHighPriorityAndTraceLevel123(HttpServletRequest request) { - if (request.getHeader(Headers.SESSION_ID) != null) { - assert (request.getHeader(Headers.PRIORITY).equals("HIGHEST")); - assert (request.getHeader(Headers.TRACE_LEVEL).equals("123")); - } - } - - private void conditionNotMetRequest(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondConditionNotMet(responseWriter, operationId); - } - closeChannel(responseWriter); - - } - private void badRequest(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - if (badRequestScenarioShouldReturnBadRequest.get()) { - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - BufferedReader reader = new BufferedReader(new InputStreamReader(request.getInputStream())); - while (reader.readLine() != null) { - //consume input, not really needed? - } - reader.close(); - closeChannel(responseWriter); - } else { - allOk(baseRequest, request, response); - } - } - - private void delayedResponse(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - if (delayedResponseShouldBlock.get()) { - CountDownLatch latch = new CountDownLatch(1); - delayedRequests.add(latch); - try { - latch.await(120, TimeUnit.SECONDS); //wait "forever" - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - if (latch.getCount() != 0L) { - throw new RuntimeException("Delayed request handler did not get poke()d."); - } - } else { - } - allOk(baseRequest, request, response); - } - - public void poke() throws InterruptedException { - CountDownLatch latch = delayedRequests.poll(10, TimeUnit.SECONDS); - latch.countDown(); - } - - public void pokeAllAndUnblockFromNowOn() { - delayedResponseShouldBlock.set(false); - while (!delayedRequests.isEmpty()) { - CountDownLatch latch = delayedRequests.remove(); - latch.countDown(); - } - } - - private void allOk(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondOK(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void wrongSessionId(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = generateMockSessionId(); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondOK(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void disconnect(Request baseRequest, HttpServletResponse response) throws IOException { - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - closeChannel(responseWriter); - } - - private void dontAcceptVersion(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(Headers.HTTP_NOT_ACCEPTABLE); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - responseWriter.write("Go away, no such version."); - responseWriter.flush(); - closeChannel(responseWriter); - } - - private void unexpectedVersion(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - response.setHeader(Headers.SESSION_ID, sessionId); - response.setHeader(Headers.VERSION, "12345678"); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondOK(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void internalServerError(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(500); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - responseWriter.write("boom"); - responseWriter.flush(); - closeChannel(responseWriter); - } - - private void couldNotFeed(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondTransientFailed(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void mbusReturnedError(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondFailedWithTransitiveErrorSeenFromClient(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void neverReturnAnyResults(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - BufferedReader reader = new BufferedReader(new InputStreamReader(request.getInputStream())); - while (reader.readLine() != null) { - //consume input, not really needed? - } - reader.close(); - closeChannel(responseWriter); - } - - void closeChannel(PrintWriter responseWriter) { - System.err.println("Mock server " + name + " closing channel."); - responseWriter.close(); - } - - private String readOperationId(InputStream requestInputStream) throws IOException { - StringBuilder idBuf = new StringBuilder(100); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 32) { - break; - } - idBuf.append((char) c); //it's ASCII - } - if (c == -1) { - return null; - } - return Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString(); - } - - private int readByteLength(InputStream requestInputStream) throws IOException { - StringBuilder lenBuf = new StringBuilder(8); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 10) { - break; - } - lenBuf.append((char) c); //it's ASCII - } - if (lenBuf.length() == 0) { - throw new IllegalStateException("Operation length missing."); - } - return Integer.valueOf(lenBuf.toString(), 16); - } - - private static void setHeaders(HttpServletResponse response, String sessionId) { - response.setHeader(Headers.SESSION_ID, sessionId); - response.setHeader(Headers.VERSION, "3"); - } - - private void respondFailed(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = - new OperationStatus("mbus returned boom", docId, ErrorCode.ERROR, false, "trace"); - writeResponse(responseWriter, operationStatus); - } - - private void respondTransientFailed(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = new OperationStatus( - "Could not put", docId, ErrorCode.TRANSIENT_ERROR, false, ""); - writeResponse(responseWriter, operationStatus); - } - - private void respondFailedWithTransitiveErrorSeenFromClient(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = - new OperationStatus("NETWORK_ERROR", docId, ErrorCode.ERROR, false, "trace"); - writeResponse(responseWriter, operationStatus); - } - - private void respondConditionNotMet(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = - new OperationStatus("this is a test", docId, ErrorCode.ERROR, true, "trace"); - writeResponse(responseWriter, operationStatus); - } - private void respondOK(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = new OperationStatus("Doc fed", docId, ErrorCode.OK, false, "Trace message"); - writeResponse(responseWriter, operationStatus); - } - - private void writeResponse(PrintWriter responseWriter, - final OperationStatus operationStatus) { - responseWriter.print(operationStatus.render()); - responseWriter.flush(); - System.err.println("Mock " + name + " server wrote: " + operationStatus.render()); - } - - private String getSessionId(HttpServletRequest request) { - return request.getHeader(Headers.CLIENT_ID); - } - - private String generateMockSessionId() { - return String.valueOf(sessionIdGenerator.getAndIncrement()); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java deleted file mode 100644 index 55f6af9ed32..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.assertThat; - -public class CommandLineArgumentsTest { - - private String[] asArray() { - String[] array = new String[args.size()]; - args.toArray(array); - return array; - } - - private void add(String key, String value) { - args.add("--" + key); - args.add(value); - } - - private void addMinimum() { - add("host", "hostValue"); - } - - private ArrayList<String> args = new ArrayList<>(); - - @Test - public void testRequiredFlags() { - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - add("file", "fileValue"); - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - args.clear(); - addMinimum(); - assertThat(CommandLineArguments.build(asArray()), is(not(nullValue()))); - } - - @Test - public void testStreaming() { - add("host", "hostValue"); - add("file", null); // Not yet implemented support for streaming - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - } - - @Test - public void testBrokenFlags() { - addMinimum(); - args.add("FOO"); - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - } - - @Test - public void testBadPriority() { - addMinimum(); - add("priority", "non existing"); - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - } - - @Test - public void testOkPriority() { - addMinimum(); - add("priority", "HIGHEST"); - assertThat(CommandLineArguments.build(asArray()).createSessionParams(false).getFeedParams().getPriority(), - is("HIGHEST")); - } - - @Test - public void testDefaults() { - addMinimum(); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(false /* use json */); - assertThat(params.getClientQueueSize(), is(10000)); - assertThat(params.getThrottlerMinSize(), is(0)); - assertThat(params.getClusters().size(), is(1)); - assertThat(params.getClusters().get(0).getEndpoints().size(), is(1)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getHostname(), is("hostValue")); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(4080)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); - assertThat(params.getConnectionParams().getUseCompression(), is(false)); - assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(4)); - assertThat(params.getFeedParams().getRoute(), is("default")); - assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.XML_UTF8)); - assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(180000L)); - assertThat(params.getFeedParams().getMaxInFlightRequests(), is(10000)); - assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(180000L)); - } - - @Test - public void testAllImplementedFlags() { - add("file", "fileValue.json"); - add("route", "routeValue"); - add("host", "hostValue"); - add("port", "1234"); - add("timeout", "2345"); - add("numPersistentConnectionsPerEndpoint", "7"); - args.add("--useCompression"); - args.add("--useDynamicThrottling"); - add("maxpending", "3456"); - args.add("--verbose"); - args.add("--useTls"); - add("header", "Header-Name: Header-Value"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - assertThat(params.getClientQueueSize(), is(3456)); - assertThat(params.getThrottlerMinSize(), is(10)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(1234)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(true)); - assertThat(params.getConnectionParams().getUseCompression(), is(true)); - assertThat(params.getConnectionParams().getHeaders().size(), is(1)); - assertThat(params.getFeedParams().getRoute(), is("routeValue")); - assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.JSON_UTF8)); - assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(2345000L)); - assertThat(params.getFeedParams().getMaxInFlightRequests(), is(3456)); - assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(2345000L)); - assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(7)); - } - - @Test - public void testAddingMultipleHttpHeaders() { - add("host", "hostValue"); - String header1Name = "Header-Name-1"; - String header1Value = "Header-Value"; - add("header", header1Name + ": " + header1Value); - String header2Name = "Header-Name-2"; - String header2Value = "Another-Header-Value"; - add("header", header2Name + ": " + header2Value); - - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - - List<Map.Entry<String, String>> headers = new ArrayList<>(params.getConnectionParams().getHeaders()); - headers.sort(Comparator.comparing(Map.Entry::getKey)); - - assertThat(headers.size(), is(2)); - Map.Entry<String, String> actualHeader1 = headers.get(0); - assertThat(actualHeader1.getKey(), is(header1Name)); - assertThat(actualHeader1.getValue(), is(header1Value)); - Map.Entry<String, String> actualHeader2 = headers.get(1); - assertThat(actualHeader2.getKey(), is(header2Name)); - assertThat(actualHeader2.getValue(), is(header2Value)); - } - - @Test - public void testMultiHost() { - add("file", "fileValue.json"); - add("port", "1234"); - add("host", "hostValue1,hostValue2, hostValue3"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - assertThat(params.getClusters().size(), is(3)); - final Set<String> hosts = new HashSet<>(); - for (Cluster cluster : params.getClusters()) { - assertThat(cluster.getEndpoints().size(), is(1)); - hosts.add(cluster.getEndpoints().get(0).getHostname()); - assertThat(cluster.getEndpoints().get(0).getPort(), is(1234)); - } - assertThat(hosts, hasItem("hostValue1")); - assertThat(hosts, hasItem("hostValue2")); - assertThat(hosts, hasItem("hostValue3")); - } - - @Test - public void testUseV3Protocol() { - addMinimum(); - args.add("--useV3Protocol"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - } - - @Test - public void testEndpoint() { - add("endpoint", "http://myendpoint:1234"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getHostname(), is("myendpoint")); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(1234)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); - } - - @Test - public void testEndpointHttps() { - add("endpoint", "https://myendpoint:1234"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(true)); - } - - @Test - public void testEndpointAndHost() { - add("host", "myhost"); - add("port", "2345"); - add("endpoint", "http://myendpoint:1234"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - assertThat(arguments, is(nullValue())); // cannot have both endpoint and host - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java deleted file mode 100644 index 9c77ed9a6d9..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.util.Optional; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -/** - * @author valerijf - */ -public class FormatInputStreamTest { - @Test(expected=IllegalArgumentException.class) - public void testWithGarbageText() throws IOException { - String streamString = "This is neither XML nor JSON!"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - } - - @Test - public void testWithFileInput() throws IOException { - String fileString = "{\"format\": \"json\"}"; - File file = File.createTempFile("feeddata", "json"); - file.deleteOnExit(); - try (FileWriter writer = new FileWriter(file)) { - writer.write(fileString); - } - - FormatInputStream formatInputStream = new FormatInputStream(null, Optional.of(file.getAbsolutePath()), false); - assertThat(fileString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON)); - } - - @Test - public void testPreferenceFileOverStream() throws IOException { - String streamString = "something entirely different"; - String fileString = "{\"format\": \"json\"}"; - - InputStream jsonStream = getInputStreamOf(streamString); - File file = File.createTempFile("feeddata", "json"); - file.deleteOnExit(); - try (FileWriter writer = new FileWriter(file)) { - writer.write(fileString); - } - - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.of(file.getAbsolutePath()), false); - assertThat(fileString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON)); - } - - @Test - public void testSimpleJsonInputStream() throws IOException { - String streamString = "{\"format\": \"json\"}"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - - assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON)); - } - - @Test - public void testSimpleXmlInputStream() throws IOException { - String streamString = "<scope><tag>format</tag><value>xml</value></scope>"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - - assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML)); - } - - @Test - public void testSparselyFormattedXml() throws IOException { - String streamString = " \t\t\n<scope>\n\n\n<tag>format</tag><value>xml</value></scope>"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - - assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML)); - } - - @Test - public void testAddRootToXml() throws IOException { - String streamString = "some random text"; - InputStream textStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(textStream, Optional.empty(), true); - - assertThat("<vespafeed>" + streamString + "</vespafeed>", - is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML)); - } - - private static String convertStreamToString(InputStream inputStream) throws IOException { - StringBuilder builder = new StringBuilder(); - while (true) { - int character = inputStream.read(); - if (character == -1) { - inputStream.close(); - return builder.toString(); - } - builder.append((char)character); - } - } - - private static InputStream getInputStreamOf(String text) { - return new ByteArrayInputStream(text.getBytes()); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java deleted file mode 100644 index 0a5b3771958..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.core.JsonReader; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.yahoo.test.json.JsonTestHelper.inputJson; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -public class JsonReaderTest { - - private static String doc1_id = "id:unittest:testMapStringToArrayOfInt::whee"; - - private static String doc1 = inputJson( - "{", - " 'update': '"+ doc1_id + "',", - " 'fields': {", - " 'actualMapStringToArrayOfInt': {", - " 'assign': [", - " { 'key': 'bamse', 'value': [ 2, 1, 3] }", - " ]", - " }", - " }", - "}"); - - private static String doc2_id = "id:unittest:smoke::whee"; - - private static String doc2 = inputJson( - "{", - " 'put': '" + doc2_id + "',", - " 'fields': {", - " 'something': 'smoketest',", - " 'nalle': 'bamse'", - " }", - "}"); - - private static String doc3 = inputJson( - "{", - " 'update': 'id:unittest:testarray::whee',", - " 'fields': {", - " 'actualarray': {", - " 'add': [", - " 'person naïve',", - " 'another person'", - " ]", - " }", - " }", - "}"); - - private static String doc4 = inputJson( - "{", - " 'remove': '" + doc2_id + "'", - "}"); - - private static String doc5_id = "id:unittest:smoking::wheels"; - - private static String doc5 = inputJson( - "{", - " 'id': '" + doc5_id + "',", - " 'fields': {", - " 'something': 'smoketest',", - " 'nalle': 'bamse'", - " }", - "}"); - - private static class TestFeedClient implements FeedClient { - - public List<String> documentIds = new ArrayList<>(); - public List<CharSequence> datas = new ArrayList<>(); - public List<Object> contexts = new ArrayList<>(); - - @Override - public void stream(String documentId, CharSequence documentData) { - stream(documentId, documentData, null); - } - - @Override - public void stream(String documentId, String operationId, CharSequence documentData, Object context) { - documentIds.add(documentId); - datas.add(documentData); - contexts.add(context); - } - - @Override - public void close() { } - - @Override - public String getStatsAsJson() { return null; } - } - - final TestFeedClient session = new TestFeedClient(); - final AtomicInteger numSent = new AtomicInteger(0); - - @Test - public void testReadNoDocument() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - " ".getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(0)); - } - - @Test - public void testReadOneDocument() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - ("["+ doc1 + "]" ).getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(1)); - assertThat(session.documentIds.get(0), is(doc1_id)); - assertThat(session.datas.size(), is(1)); - assertThat(session.datas.get(0), is(doc1)); - } - - @Test - public void testReadFourDocuments() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - (" [ "+ doc1 + " , " + doc2 + ", " + doc3 + "," + doc4 + " ] ").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(4)); - assertThat(session.documentIds.get(0), is(doc1_id)); - assertThat(session.documentIds.get(1), is(doc2_id)); - assertThat(session.datas.size(), is(4)); - assertThat(session.datas.get(0), is(doc1)); - assertThat(session.datas.get(1).toString(), is(doc2)); - assertThat(session.datas.get(2).toString(), is(doc3)); - assertThat(session.datas.get(3).toString(), is(doc4)); - } - - @Test - public void testDocWithIdAndNotPut() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - (" [ "+ doc5 + " ] ").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(1)); - assertThat(session.documentIds.get(0), is(doc5_id)); - } - - @Test - public void simpleMicroBenchmarkTest() throws Exception { - StringBuilder stream = new StringBuilder(); - stream.append("["); - int docsInStream = 15000; - for (int x = 0; x < docsInStream -1; x++) { - if (x % 10 == 0) { - stream.append(doc1 + ", "); - } else { - // Add some randomness to the layout to trigger potential bugs in parsing. - stream.append("{\"remove\": \"id:unittest:smoke::whee"); - for (int y = 0 ; y < x % 277 ; y++) { - stream.append("X"); - } - stream.append("\"}, "); - } - } - stream.append(doc3); - stream.append("]"); - - InputStream inputStream = new ByteArrayInputStream(stream.toString().getBytes(StandardCharsets.UTF_8)); - long startTime = System.currentTimeMillis(); - JsonReader.read(inputStream, session, numSent); - // At time of writing, it took about 200 ms on my mac. - System.err.println("Run time is " + (System.currentTimeMillis() - startTime) + " ms"); - inputStream.close(); - - // Verify that content is not rubbish. - for (int x = 0; x < docsInStream - 1; x++) { - if (x % 10 == 0) { - assertThat(session.datas.get(x).toString(), is(doc1)); - assertThat(session.documentIds.get(x), is(doc1_id)); - } - } - assertThat(session.datas.get(docsInStream-1).toString(), is(doc3)); - assertThat(numSent.get(), is(docsInStream)); - } - - @Test(expected=RuntimeException.class) - public void testBadJsonCommaAfterLastElement() { - InputStream inputStream = new ByteArrayInputStream( - ("["+ doc1 + ",]" ).getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - } - - @Test(expected=RuntimeException.class) - public void testTotalGarbage() { - InputStream inputStream = new ByteArrayInputStream(("garbage" ).getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - } - - @Test(expected=RuntimeException.class) - public void testTwoDocIds() { - InputStream inputStream = new ByteArrayInputStream(("[{\"remove\": \"id\", \"update\": \"id:\"}]" - .getBytes(StandardCharsets.UTF_8))); - JsonReader.read(inputStream, session, numSent); - } - - @Test(expected = IllegalArgumentException.class) - public void throwsOnMissingId() { - InputStream inputStream = new ByteArrayInputStream( - inputJson("[{'fields':{ 'something': 'smoketest', 'nalle': 'bamse' }}]").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - } - - @Test - public void testFullDocument() throws Exception { - InputStream inputStream = new ByteArrayInputStream(( - "[{\n" + - " \"update\": \"id:foo:music:doc:foo:bar\",\n" + - " \n" + - " \"fields\": {\n" + - " \"artist\": {\n" + - " \"assign\": null" + - " },\n" + - " \n" + - " \"albums\": {\n" + - " \"assign\": [\n" + - " \"Kramgoda laatar 4\",\n" + - " \"Kramgoda laatar 5\",\n" + - " \"Kramgoda laatar 6\"\n" + - " ],\n" + - " \"add\": [\n" + - " \"Kramgoda laatar 7\",\n" + - " \"Kramgoda laatar 8\"\n" + - " ]\n" + - " },\n" + - " \"inceptionYear\": {\n" + - " \"increment\": 4\n" + - " },\n" + - " \"concerts\": {\n" + - " \"assign\": {\n" + - " \"Torsby 1993\": 1000,\n" + - " \"Uddevalla 2000\": 34\n" + - " },\n" + - " \"match\": {\n" + - " \"element\": \"Sundsvall 1980\",\n" + - " \"increment\": 5392\n" + - " },\n" + - " \"add\": {\n" + - " \"Kiruna 1999\": 200,\n" + - " \"Oslo 1998\": 2000\n" + - " }\n" + - " },\n" + - " \"scores\": {\n" + - " \"match\": {\n" + - " \"element\": \"Sven Ingvars\",\n" + - " \"match\": {\n" + - " \"element\": 0,\n" + - " \"increment\": 78\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}]\n").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(1)); - assertThat(session.documentIds.get(0), is("id:foo:music:doc:foo:bar")); - } -} diff --git a/vespa-http-client/src/test/resources/vespacorpfeed-prod-sample.xml b/vespa-http-client/src/test/resources/vespacorpfeed-prod-sample.xml deleted file mode 100644 index 95f0e4b1961..00000000000 --- a/vespa-http-client/src/test/resources/vespacorpfeed-prod-sample.xml +++ /dev/null @@ -1,187 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::7138bc793a096a78b86a6501ae0c6e7b"> - <threadId>f5078d76c7541ab15387ab62fef22a01</threadId> - <contentsHash>186e631cb3b09ac33e5c124a79e20915</contentsHash> - <authors> - <item>bratseth</item> - </authors> - <urlPath>/vespa-users/msg/4498622c3f131e1067c4e7bc18ac96db</urlPath> - <contents>This was done to make it simpler for people to get started with jars loaded through OSGi on the assumption that most wouldn't need to import anything I'm not sure it actually gives a net saving though for the reason you point out We'll probably make the existence of global packages optional somehow later Jon Den 21 mai 2011 kl 03.21 skrev LG Just out of curiosity why motivated the choice of exposing them as global packages instead of exported packages I had personally a hard time figuring out which packages I had to import and to not import javax com.yahoo.vespa etc and I'm wondering if it means I need to repackage some 3rdparty component if they decide to import some of these global packages like javax.foo lg geoinformatics software engineer mail@host.com direct y!im 701 first avenue sunnyvale ca 94089-0703 us phone fax Le 5/10/11 12:10 AM Jon S Bratseth a écrit The packages in the Vespa public api is always available global That's the ones included here http://vespa/javadoc/5.0/all In addition the packages belonging to the Java SE API javax etc are global The exported list is not quite as well defined though You can always just try to import when some package is not in the global category above if it cannot be resolved then we do not export it Or check the source as Francois mentions </contents> - <parentId>8bce4afebf9dca1aab84651db62d4269</parentId> - <title>How to load JSON library in vespa 5.0.8</title> - <my_contents>This was done to make it simpler for people to get started with jars loaded through OSGi on the assumption that most wouldn't need to import anything I'm not sure it actually gives a net saving though for the reason you point out We'll probably make the existence of global packages optional somehow later Jon Den 21 mai 2011 kl 03.21 skrev LG Just out of curiosity why motivated the choice of exposing them as global packages instead of exported packages I had personally a hard time figuring out which packages I had to import and to not import javax com.yahoo.vespa etc and I'm wondering if it means I need to repackage some 3rdparty component if they decide to import some of these global packages like javax.foo laurent goujon geoinformatics software engineer mail@host.com direct y!im 701 first avenue sunnyvale ca 94089-0703 us phone fax Le 5/10/11 12:10 AM Jon S Bratseth a écrit The packages in the Vespa public api is always available global That's the ones included here http://vespa/javadoc/5.0/all In addition the packages belonging to the Java SE API javax etc are global The exported list is not quite as well defined though You can always just try to import when some package is not in the global category above if it cannot be resolved then we do not export it Or check the source as Francois mentions </my_contents> - <lastUpdate>1306231886</lastUpdate> - <pagerank>0</pagerank> - <date>1306228138</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>How to load JSON library in vespa 5.0.8</my_title> - <messageId>4498622c3f131e1067c4e7bc18ac96db</messageId> - <origContents binaryencoding="base64">VGhpcyB3YXMgZG9uZSB0byBtYWtlIGl0IHNpbXBsZXIgZm9yIHBlb3BsZSB0byBnZXQgc3RhcnRlZCB3aXRoIGphcnMgbG9hZGVkIHRocm91Z2ggT1NHaSwgb24gdGhlIGFzc3VtcHRpb24gdGhhdCBtb3N0IHdvdWxkbid0IG5lZWQgdG8gaW1wb3J0IGFueXRoaW5nLgpJJ20gbm90IHN1cmUgaXQgYWN0dWFsbHkgZ2l2ZXMgYSBuZXQgc2F2aW5nIHRob3VnaCwgZm9yIHRoZSByZWFzb24geW91IHBvaW50IG91dC4gV2UnbGwgcHJvYmFibHkgbWFrZSB0aGUgZXhpc3RlbmNlIG9mIGdsb2JhbCBwYWNrYWdlcyBvcHRpb25hbCBzb21laG93IGxhdGVyLgoKLS0KSm9uCgpEZW4gMjEuIG1haSAyMDExIGtsLiAwMy4yMSBza3JldiBMYXVyZW50IEdvdWpvbjoKCkp1c3Qgb3V0IG9mIGN1cmlvc2l0eSwgd2h5IG1vdGl2YXRlZCB0aGUgY2hvaWNlIG9mIGV4cG9zaW5nIHRoZW0gYXMgZ2xvYmFsIHBhY2thZ2VzIGluc3RlYWQgb2YgZXhwb3J0ZWQgcGFja2FnZXM/CgpJIGhhZCBwZXJzb25hbGx5IGEgaGFyZCB0aW1lIGZpZ3VyaW5nIG91dCB3aGljaCBwYWNrYWdlcyBJIGhhZCB0byBpbXBvcnQgYW5kIHRvIG5vdCBpbXBvcnQgKGphdmF4LiosIGNvbS55YWhvby52ZXNwYSwgZXRjLi4uKSBhbmQgSSdtIHdvbmRlcmluZyBpZiBpdCBtZWFucyBJIG5lZWQgdG8gcmVwYWNrYWdlIHNvbWUgM3JkcGFydHkgY29tcG9uZW50IGlmIHRoZXkgZGVjaWRlIHRvIGltcG9ydCBzb21lIG9mIHRoZXNlIGdsb2JhbCBwYWNrYWdlcyBsaWtlIGphdmF4LmZvby4KCmxhdXJlbnQKZ291am9uCgpnZW9pbmZvcm1hdGljcyBzb2Z0d2FyZSBlbmdpbmVlcgoKZ291am9ubEB5YWhvby1pbmMuY29tPG1haWx0bzpnb3Vqb25sQHlhaG9vLWluYy5jb20+CmRpcmVjdCAoNDA4KSAzNDkgOTMwMgp5IWltIHlsYXVyZW50Z28KCjcwMSBmaXJzdCBhdmVudWUsIHN1bm55dmFsZSwgY2EsIDk0MDg5LTA3MDMsIHVzCnBob25lICg0MDgpIDM0OSAzMzAwICAgIGZheCAoNDA4KSAzNDkgMzMwMQoKPHlhaG9vLmdpZj4KCgpMZSA1LzEwLzExIDEyOjEwIEFNLCBKb24gUyBCcmF0c2V0aCBhIMOpY3JpdCA6CgpUaGUgcGFja2FnZXMgaW4gdGhlIFZlc3BhIHB1YmxpYyBhcGkgaXMgYWx3YXlzIGF2YWlsYWJsZSAoZ2xvYmFsKS4KVGhhdCdzIHRoZSBvbmVzIGluY2x1ZGVkIGhlcmU6IGh0dHA6Ly92ZXNwYS5jb3JwLnlhaG9vLmNvbS9qYXZhZG9jLzUuMC9hbGwvCkluIGFkZGl0aW9uLCB0aGUgcGFja2FnZXMgYmVsb25naW5nIHRvIHRoZSBKYXZhIFNFIEFQSSAoamF2YXggZXRjLikgYXJlIGdsb2JhbC4KClRoZSAiZXhwb3J0ZWQiIGxpc3QgaXMgbm90IHF1aXRlIGFzIHdlbGwgZGVmaW5lZCB0aG91Z2guIFlvdSBjYW4gYWx3YXlzIGp1c3QgdHJ5IHRvIGltcG9ydCB3aGVuIHNvbWUgcGFja2FnZSBpcyBub3QgaW4gdGhlICJnbG9iYWwiIGNhdGVnb3J5IGFib3ZlLCBpZiBpdCBjYW5ub3QgYmUgcmVzb2x2ZWQgdGhlbiB3ZSBkbyBub3QgZXhwb3J0IGl0LiBPciBjaGVjayB0aGUgc291cmNlIGFzIEZyYW5jb2lzIG1lbnRpb25zLg==</origContents> - <threadUrl>/2011/05/09/how_to_load_json_library_in_vespa_5_0_8</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::11b6e7f86d650b995289dcadaaf902fc"> - <threadId>f4467ca341409984b9c3ab4c431f0a7c</threadId> - <contentsHash>c0d00a07cc1486404f5e23a2cc3cc5ed</contentsHash> - <authors> - <item>bratseth</item> - </authors> - <urlPath>/vespa-users/msg/2538973d3c06d9d5163bf9bd069d0bd6</urlPath> - <contents>The unit test and doc I pointed to had some examples were you looking for something else The unit tests are using processors from this set http://vespa/view/vespa/trunk/container/processing/src/main/java/com/yahoo/processing/test/ProcessorLibrary.java?view=markup From this you might be interested in Federator forks the execution to multiple chains executed in parallel FutureDataSource creates and returns response containing future data BlockingSplitter An example of waiting for some future data to complete before using them for some processing of course don t do this if you want full async for some reason AsyncDataProcessingInitiator An example of registering future processing when a list of data is completed i.e process the data without blocking StreamProcessingInitiator An example of registering future processing on every additional piece of data entering a list None of this shows how to create a real async data source which gets its data from the network the FutureDataSource above is of course just a mock I have an example of that somewhere which I can dig up if you need it Jon On 6 feb 2014 at 02:40 GOs wrote Is there an example which uses the futures We have async working but it's not sending all of the data until the request is marked as completed Best GGO Tech Yahoo Software Systems Development Engineer M 701 First Avenue Sunnyvale CA 94089 http://forgood.zenfs.com/logos/yahoo.png On Wednesday February 5 2014 2:02 AM Jon Bratseth wrote See the processing framework in particular http://vespa/5/documentation/jdisc/processing.html#asynchronous-processing and the example AsyncDataProducer towards the end of the page In short the Processing framework supports this use case out of the box just return a Response with data futures and the renderer will render as much as possible at once and then the future data whenever it becomes available See http://vespa/view/vespa/trunk/container/core/src/test/java/com/yahoo/processing/handler/ProcessingHandlerTestCase.java?view=markup for some complete examples the tests named something with async Also note that by default the renderer will preserve the order of the future placeholders in the response such that if a Response containing future data list A and B it will wait for A before rendering further even if B is available If you want to return data as soon as possible in any order have your DataList instances implement com.yahoo.processing.response.Ordered and return false from isOrdered There s an example of that as well in the test above Jon On 5 feb 2014 at 10:43 Kristian Aune wrote vespa-users is the best list K On 5 feb 2014 at 01:24 GO wrote Hey Kristian Not sure what the ilist is for JDisc user support so if you know just reply and I'll forward this mail that way Basically here's the scenario I'm looking at We get a request from client and receive it in JDisc Then we do a bunch of processing and serve responses as they come this way the client can immediately render finished responses while still waiting for pending responses What is the easiest way to do this in a JDisc container I was looking at websockets but not sure how much they are supported Best GE desde mi iPhone Vespa Information http://vespa </contents> - <parentId>1c809fe04d3807dc72be71cf1aa559d6</parentId> - <title>JDisc streaming / websockets</title> - <my_contents>The unit test and doc I pointed to had some examples were you looking for something else The unit tests are using processors from this set http://vespa/view/vespa/trunk/container/processing/src/main/java/com/yahoo/processing/test/ProcessorLibrary.java?view=markup From this you might be interested in Federator forks the execution to multiple chains executed in parallel FutureDataSource creates and returns response containing future data BlockingSplitter An example of waiting for some future data to complete before using them for some processing of course don t do this if you want full async for some reason AsyncDataProcessingInitiator An example of registering future processing when a list of data is completed i.e process the data without blocking StreamProcessingInitiator An example of registering future processing on every additional piece of data entering a list None of this shows how to create a real async data source which gets its data from the network the FutureDataSource above is of course just a mock I have an example of that somewhere which I can dig up if you need it Jon On 6 feb 2014 at 02:40 GO wrote Is there an example which uses the futures We have async working but it's not sending all of the data until the request is marked as completed Best GGO Tech Yahoo Software Systems Development Engineer MFirst Avenue Sunnyvale CA 94089 http://forgood.zenfs.com/logos/yahoo.png On Wednesday February 5 2014 2:02 AM Jon Bratseth wrote See the processing framework in particular http://vespa/5/documentation/jdisc/processing.html#asynchronous-processing and the example AsyncDataProducer towards the end of the page In short the Processing framework supports this use case out of the box just return a Response with data futures and the renderer will render as much as possible at once and then the future data whenever it becomes available See http://vespa/view/vespa/trunk/container/core/src/test/java/com/yahoo/processing/handler/ProcessingHandlerTestCase.java?view=markup for some complete examples the tests named something with async Also note that by default the renderer will preserve the order of the future placeholders in the response such that if a Response containing future data list A and B it will wait for A before rendering further even if B is available If you want to return data as soon as possible in any order have your DataList instances implement com.yahoo.processing.response.Ordered and return false from isOrdered There s an example of that as well in the test above Jon On 5 feb 2014 at 10:43 Kristian Aune wrote vespa-users is the best list K On 5 feb 2014 at 01:24 GO wrote Hey Kristian Not sure what the ilist is for JDisc user support so if you know just reply and I'll forward this mail that way Basically here's the scenario I'm looking at We get a request from client and receive it in JDisc Then we do a bunch of processing and serve responses as they come this way the client can immediately render finished responses while still waiting for pending responses What is the easiest way to do this in a JDisc container I was looking at websockets but not sure how much they are supported Best GE desde mi iPhone Vespa Information http://vespa</my_contents> - <lastUpdate>1391683805</lastUpdate> - <pagerank>0</pagerank> - <date>1391680103</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>JDisc streaming / websockets</my_title> - <messageId>2538973d3c06d9d5163bf9bd069d0bd6</messageId> - <origContents binaryencoding="base64">VGhlIHVuaXQgdGVzdCBhbmQgZG9jIEkgcG9pbnRlZCB0byBoYWQgc29tZSBleGFtcGxlcywgd2VyZSB5b3UgbG9va2luZyBmb3Igc29tZXRoaW5nIGVsc2U/CgpUaGUgdW5pdCB0ZXN0cyBhcmUgdXNpbmcgcHJvY2Vzc29ycyBmcm9tIHRoaXMgc2V0OgpodHRwOi8vc3ZuLnRyb25kaGVpbS5jb3JwLnlhaG9vLmNvbS92aWV3L3Zlc3BhL3RydW5rL2NvbnRhaW5lci9wcm9jZXNzaW5nL3NyYy9tYWluL2phdmEvY29tL3lhaG9vL3Byb2Nlc3NpbmcvdGVzdC9Qcm9jZXNzb3JMaWJyYXJ5LmphdmE/dmlldz1tYXJrdXAKRnJvbSB0aGlzLiB5b3UgbWlnaHQgYmUgaW50ZXJlc3RlZCBpbgpGZWRlcmF0b3IgLSBmb3JrcyB0aGUgZXhlY3V0aW9uIHRvIG11bHRpcGxlIGNoYWlucyBleGVjdXRlZCBpbiBwYXJhbGxlbC4KRnV0dXJlRGF0YVNvdXJjZSAtIGNyZWF0ZXMgYW5kIHJldHVybnMgcmVzcG9uc2UgY29udGFpbmluZyBmdXR1cmUgZGF0YS4KQmxvY2tpbmdTcGxpdHRlciAtIEFuIGV4YW1wbGUgb2Ygd2FpdGluZyBmb3Igc29tZSBmdXR1cmUgZGF0YSB0byBjb21wbGV0ZSBiZWZvcmUgdXNpbmcgdGhlbSBmb3Igc29tZSBwcm9jZXNzaW5nIChvZiwgY291cnNlLCBkb27CknQgZG8gdGhpcyBpZiB5b3Ugd2FudCBmdWxsIGFzeW5jIGZvciBzb21lIHJlYXNvbikKQXN5bmNEYXRhUHJvY2Vzc2luZ0luaXRpYXRvciAtIEFuIGV4YW1wbGUgb2YgcmVnaXN0ZXJpbmcgZnV0dXJlIHByb2Nlc3Npbmcgd2hlbiBhIGxpc3Qgb2YgZGF0YSBpcyBjb21wbGV0ZWQgKGkuZSBwcm9jZXNzIHRoZSBkYXRhIHdpdGhvdXQgYmxvY2tpbmcpClN0cmVhbVByb2Nlc3NpbmdJbml0aWF0b3IgLSBBbiBleGFtcGxlIG9mIHJlZ2lzdGVyaW5nIGZ1dHVyZSBwcm9jZXNzaW5nIG9uIGV2ZXJ5IGFkZGl0aW9uYWwgcGllY2Ugb2YgZGF0YSBlbnRlcmluZyBhIGxpc3QKCk5vbmUgb2YgdGhpcyBzaG93cyBob3cgdG8gY3JlYXRlIGEgKnJlYWwqIGFzeW5jIGRhdGEgc291cmNlIHdoaWNoIGdldHMgaXRzIGRhdGEgZnJvbSB0aGUgbmV0d29yayAodGhlIEZ1dHVyZURhdGFTb3VyY2UgYWJvdmUgaXMgb2YgY291cnNlIGp1c3QgYSBtb2NrKS4KSSBoYXZlIGFuIGV4YW1wbGUgb2YgdGhhdCBzb21ld2hlcmUgd2hpY2ggSSBjYW4gZGlnIHVwIGlmIHlvdSBuZWVkIGl0LgoKwpcKSm9uCgpPbiA2LiBmZWIuIDIwMTQsIGF0IDAyOjQwLCBHYXZpbiBPd2VucyA8Z293ZW5zQHlhaG9vLWluYy5jb208bWFpbHRvOmdvd2Vuc0B5YWhvby1pbmMuY29tPj4gd3JvdGU6CgpJcyB0aGVyZSBhbiBleGFtcGxlIHdoaWNoIHVzZXMgdGhlIGZ1dHVyZXM/CgpXZSBoYXZlIGFzeW5jIHdvcmtpbmcsIGJ1dCBpdCdzIG5vdCBzZW5kaW5nIGFsbCBvZiB0aGUgZGF0YSB1bnRpbCB0aGUgcmVxdWVzdCBpcyBtYXJrZWQgYXMgY29tcGxldGVkLgoKQmVzdCwKR2F2aW4KCgpHYXZpbiBPd2VucwpUZWNoIFlhaG9vLCBTb2Z0d2FyZSBTeXN0ZW1zIERldmVsb3BtZW50IEVuZ2luZWVyCk06ICg0MDgpIDMwNi03NTM2CjcwMSBGaXJzdCBBdmVudWUKU3Vubnl2YWxlIENBIDk0MDg5CgpbaHR0cDovL2Zvcmdvb2QuemVuZnMuY29tL2xvZ29zL3lhaG9vLnBuZ10KCgpPbiBXZWRuZXNkYXksIEZlYnJ1YXJ5IDUsIDIwMTQgMjowMiBBTSwgSm9uIEJyYXRzZXRoIDxicmF0c2V0aEB5YWhvby1pbmMuY29tPG1haWx0bzpicmF0c2V0aEB5YWhvby1pbmMuY29tPj4gd3JvdGU6ClNlZSB0aGUgcHJvY2Vzc2luZyBmcmFtZXdvcmssIGluIHBhcnRpY3VsYXIgaHR0cDovL3Zlc3BhLmNvcnAueWFob28uY29tLzUvZG9jdW1lbnRhdGlvbi9qZGlzYy9wcm9jZXNzaW5nLmh0bWwjYXN5bmNocm9ub3VzLXByb2Nlc3NpbmcKYW5kIHRoZSBleGFtcGxlIEFzeW5jRGF0YVByb2R1Y2VyIHRvd2FyZHMgdGhlIGVuZCBvZiB0aGUgcGFnZS4KCkluIHNob3J0LCB0aGUgUHJvY2Vzc2luZyBmcmFtZXdvcmsgc3VwcG9ydHMgdGhpcyB1c2UgY2FzZSBvdXQgb2YgdGhlIGJveCAtIGp1c3QgcmV0dXJuIGEgUmVzcG9uc2Ugd2l0aCBkYXRhIGZ1dHVyZXMgYW5kIHRoZSByZW5kZXJlciB3aWxsIHJlbmRlciBhcyBtdWNoIGFzIHBvc3NpYmxlIGF0IG9uY2UgYW5kIHRoZW4gdGhlIGZ1dHVyZSBkYXRhIHdoZW5ldmVyIGl0IGJlY29tZXMgYXZhaWxhYmxlLgoKClNlZSBodHRwOi8vc3ZuLnRyb25kaGVpbS5jb3JwLnlhaG9vLmNvbS92aWV3L3Zlc3BhL3RydW5rL2NvbnRhaW5lci9jb3JlL3NyYy90ZXN0L2phdmEvY29tL3lhaG9vL3Byb2Nlc3NpbmcvaGFuZGxlci9Qcm9jZXNzaW5nSGFuZGxlclRlc3RDYXNlLmphdmE/dmlldz1tYXJrdXAKZm9yIHNvbWUgY29tcGxldGUgZXhhbXBsZXMgKHRoZSB0ZXN0cyBuYW1lZCBzb21ldGhpbmcgd2l0aCBhc3luYykuCgoKQWxzbywgbm90ZSB0aGF0IGJ5IGRlZmF1bHQgdGhlIHJlbmRlcmVyIHdpbGwgcHJlc2VydmUgdGhlIG9yZGVyIG9mIHRoZSBmdXR1cmUgcGxhY2Vob2xkZXJzIGluIHRoZSByZXNwb25zZSwgc3VjaCB0aGF0IGlmIGEgUmVzcG9uc2UgY29udGFpbmluZyBmdXR1cmUgZGF0YSBsaXN0IEEgYW5kIEIsIGl0IHdpbGwgd2FpdCBmb3IgQSBiZWZvcmUgcmVuZGVyaW5nIGZ1cnRoZXIgZXZlbiBpZiBCIGlzIGF2YWlsYWJsZS4KSWYgeW91IHdhbnQgdG8gcmV0dXJuIGRhdGEgYXMgc29vbiBhcyBwb3NzaWJsZSBpbiBhbnkgb3JkZXIsIGhhdmUgeW91ciBEYXRhTGlzdCBpbnN0YW5jZXMgaW1wbGVtZW50IGNvbS55YWhvby5wcm9jZXNzaW5nLnJlc3BvbnNlLk9yZGVyZWQgYW5kIHJldHVybiBmYWxzZSBmcm9tIGlzT3JkZXJlZC4gVGhlcmXCknMgYW4gZXhhbXBsZSBvZiB0aGF0IGFzIHdlbGwgaW4gdGhlIHRlc3QgYWJvdmUuCgrClwpKb24KCk9uIDUuIGZlYi4gMjAxNCwgYXQgMTA6NDMsIEtyaXN0aWFuIEF1bmUgPEtyaXN0aWFuLkF1bmVAeWFob28taW5jLmNvbTxtYWlsdG86S3Jpc3RpYW4uQXVuZUB5YWhvby1pbmMuY29tPj4gd3JvdGU6Cgo+IHZlc3BhLXVzZXJzQCBpcyB0aGUgYmVzdCBsaXN0IDstKQo+Cj4gLUsKPgo+IE9uIDUuIGZlYi4gMjAxNCwgYXQgMDE6MjQsIEdhdmluIE93ZW5zIDxnb3dlbnNAeWFob28taW5jLmNvbTxtYWlsdG86Z293ZW5zQHlhaG9vLWluYy5jb20+PiB3cm90ZToKPgo+PiBIZXkgS3Jpc3RpYW4sCj4+Cj4+IE5vdCBzdXJlIHdoYXQgdGhlIGlsaXN0IGlzIGZvciBKRGlzYyB1c2VyIHN1cHBvcnQsIHNvIGlmIHlvdSBrbm93IGp1c3QgcmVwbHkgYW5kIEknbGwgZm9yd2FyZCB0aGlzIG1haWwgdGhhdCB3YXkuCj4+Cj4+IEJhc2ljYWxseSwgaGVyZSdzIHRoZSBzY2VuYXJpbyBJJ20gbG9va2luZyBhdC4uLgo+Pgo+PiBXZSBnZXQgYSByZXF1ZXN0IGZyb20gY2xpZW50IGFuZCByZWNlaXZlIGl0IGluIEpEaXNjLiBUaGVuIHdlIGRvIGEgYnVuY2ggb2YgcHJvY2Vzc2luZyBhbmQgc2VydmUgcmVzcG9uc2VzIGFzIHRoZXkgY29tZSAodGhpcyB3YXkgdGhlIGNsaWVudCBjYW4gaW1tZWRpYXRlbHkgcmVuZGVyIGZpbmlzaGVkIHJlc3BvbnNlcyB3aGlsZSBzdGlsbCB3YWl0aW5nIGZvciBwZW5kaW5nIHJlc3BvbnNlcykuCj4+Cj4+IFdoYXQgaXMgdGhlIGVhc2llc3Qgd2F5IHRvIGRvIHRoaXMgaW4gYSBKRGlzYyBjb250YWluZXI/IEkgd2FzIGxvb2tpbmcgYXQgd2Vic29ja2V0cyBidXQgbm90IHN1cmUgaG93IG11Y2ggdGhleSBhcmUgc3VwcG9ydGVkLgo+Pgo+PiBCZXN0LAo+PiBHYXZpbgo+Pgo+PiBFbnZpYWRvIGRlc2RlIG1pIGlQaG9uZQoKPgo+Cj4KPiBWZXNwYSBJbmZvcm1hdGlvbjoKPiAgICBodHRwOi8vdmVzcGEuY29ycC55YWhvby5jb20vCj4gICAgaHR0cDovL3R3aWtpLmNvcnAueWFob28uY29tL3ZpZXcvVmVzcGEKCj4=</origContents> - <threadUrl>/2014/02/05/jdisc_streaming_websockets</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::9de81494755bf8ee6940bcdf156081e2"> - <threadId>39e8f15ad422399d1c72d3f178f956c2</threadId> - <contentsHash>d9d95e6364aaac73b3bfb77f377236fc</contentsHash> - <urlPath>/vespa-users/msg/4f12250dae5ee19458973a0afaf6967e</urlPath> - <contents>I don't get any error while deployment the expression using created_at directly I have filed a support BZ 3455702 for this Thanks Yi On Mar 7 2010 at 10:23 AM Jo Kristian Bergum wrote On Mar 5 2010 at 11:47 PM YZ wrote Hi folks I'm trying to confirm the range of value for nativeRank is between 0 and 1 With this rank profile rank-profile native first-phase expression nativeRank I get the following result Question is relevancy showing the ranking score If so why it's 0 Depends on your query what fields were searched and your search definition I did another test this time using rank-profile created_at first-phase expression created_at This should have failed during deploy correct is expression attribute(created_at You probably have quite a few failed blueprint compilation errors in the vespa.log on the search nodes The result is as follows relevancy is still 0 and I would expect the value of created_at which is 1267569885 1267569885 Thanks Y </contents> - <parentId>df5b35d538401b06cfbe803b55856a3e</parentId> - <title>Value of nativeRank</title> - <my_contents>I don't get any error while deployment the expression using created_at directly I have filed a support BZ 3455702 for this Thanks Yi On Mar 7 2010 at 10:23 AM Jo Kristian Bergum wrote On Mar 5 2010 at 11:47 PM YZ wrote Hi folks I'm trying to confirm the range of value for nativeRank is between 0 and 1 With this rank profile rank-profile native first-phase expression nativeRank I get the following result Question is relevancy showing the ranking score If so why it's 0 Depends on your query what fields were searched and your search definition I did another test this time using rank-profile created_at first-phase expression created_at This should have failed during deploy correct is expression attribute(created_at You probably have quite a few failed blueprint compilation errors in the vespa.log on the search nodes The result is as follows relevancy is still 0 and I would expect the value of created_at which is 1267569885 1267569885 Thanks Y </my_contents> - <lastUpdate>1267989987</lastUpdate> - <pagerank>0</pagerank> - <date>1267986312</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>Value of nativeRank</my_title> - <messageId>4f12250dae5ee19458973a0afaf6967e</messageId> - <origContents binaryencoding="base64">SSBkb24ndCBnZXQgYW55IGVycm9yIHdoaWxlIGRlcGxveW1lbnQgdGhlIGV4cHJlc3Npb24gdXNpbmcgY3JlYXRlZF9hdCBkaXJlY3RseS4KSSBoYXZlIGZpbGVkIGEgc3VwcG9ydCBCWiAzNDU1NzAyIGZvciB0aGlzLgoKVGhhbmtzLAotWWkKCk9uIE1hciA3LCAyMDEwLCBhdCAxMDoyMyBBTSwgSm8gS3Jpc3RpYW4gQmVyZ3VtIHdyb3RlOgoKCk9uIE1hciA1LCAyMDEwLCBhdCAxMTo0NyBQTSwgWWkgWmhhbmcgd3JvdGU6Cgo+IEhpIGZvbGtzLAo+Cj4gICBJJ20gdHJ5aW5nIHRvIGNvbmZpcm0gdGhlIHJhbmdlIG9mIHZhbHVlIGZvciBuYXRpdmVSYW5rIGlzIGJldHdlZW4KPiAwIGFuZCAxLgo+Cj4gICBXaXRoIHRoaXMgcmFuayBwcm9maWxlOgo+ICAgcmFuay1wcm9maWxlIG5hdGl2ZXsKPiAgICAgZmlyc3QtcGhhc2UgeyBleHByZXNzaW9uOiBuYXRpdmVSYW5rIH0KPiAgIH0KPiAgIEkgZ2V0IHRoZSBmb2xsb3dpbmcgcmVzdWx0Ogo+Cj4gPHJlc3VsdCB0b3RhbC1oaXQtY291bnQ9IjIiPgo+IDxoaXQgcmVsZXZhbmN5PSIwIiBzb3VyY2U9InNjMC5udW0wIj4uLi4KPgo+ICAgUXVlc3Rpb246IGlzICJyZWxldmFuY3kiIHNob3dpbmcgdGhlIHJhbmtpbmcgc2NvcmU/IElmIHNvLCB3aHkgaXQncwo+IDA/CgpEZXBlbmRzIG9uIHlvdXIgcXVlcnkgKHdoYXQgZmllbGRzIHdlcmUgc2VhcmNoZWQpIGFuZCB5b3VyIHNlYXJjaApkZWZpbml0aW9uLgoKCj4gICBJIGRpZCBhbm90aGVyIHRlc3QsIHRoaXMgdGltZSB1c2luZwo+ICAgcmFuay1wcm9maWxlIGNyZWF0ZWRfYXR7Cj4gICAgIGZpcnN0LXBoYXNlIHsgIGV4cHJlc3Npb246IGNyZWF0ZWRfYXQgfQo+ICAgfQoKVGhpcyBzaG91bGQgaGF2ZSBmYWlsZWQgZHVyaW5nIGRlcGxveSwgY29ycmVjdCBpcwoKZXhwcmVzc2lvbjogYXR0cmlidXRlKGNyZWF0ZWRfYXQpCgpZb3UgcHJvYmFibHkgaGF2ZSBxdWl0ZSBhIGZldyAiZmFpbGVkIGJsdWVwcmludCBjb21waWxhdGlvbiIgZXJyb3JzIGluCnRoZSB2ZXNwYS5sb2cgb24gdGhlIHNlYXJjaCBub2Rlcz8KCgo+ICAgVGhlIHJlc3VsdCBpcyBhcyBmb2xsb3dzLCByZWxldmFuY3kgaXMgc3RpbGwgMCwgYW5kIEkgd291bGQgZXhwZWN0Cj4gdGhlIHZhbHVlIG9mIGNyZWF0ZWRfYXQsIHdoaWNoIGlzIDEyNjc1Njk4ODUuCj4gPHJlc3VsdCB0b3RhbC1oaXQtY291bnQ9IjIiPgo+IDxoaXQgcmVsZXZhbmN5PSIwIiBzb3VyY2U9InNjMC5udW0wIj4KPiA8ZmllbGQgbmFtZT0iY3JlYXRlZF9hdCI+MTI2NzU2OTg4NTwvZmllbGQ+Cj4KPiBUaGFua3MsCj4gLVlpCj4gPG1lc3NhZ2UtZm9vdGVyLnR4dD4=</origContents> - <threadUrl>/2010/03/05/value_of_nativerank</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::25ec8cc793e7e49cee2b5584917f6006"> - <threadId>1a60037c3896299b0a90dd5d62034529</threadId> - <contentsHash>7d08e2ecdc2a7d243a748059db0e09ee</contentsHash> - <authors> - <item>bratseth</item> - </authors> - <urlPath>/vespa-users/msg/3eac9311090c9cf7c34f9d8c897f2c4b</urlPath> - <contents> search=incr&restrict=abc RJ skrev Hi Jo We have multiple SDs deployed in one cluster and we normally use search= in our queries How do we use it for differentiating both the cluster name and the sd name For example We have two clusters namely incr and realtime each having SDs abc xyz We need a way to query from incr cluster in abc sd Thanks RJ Jo Kristian Bergum wrote On Mon 2009-08-24 at 16:55 +0530 AS wrote On the serving side how do we indicate the cluster to get results from search= /JKB </contents> - <parentId>00b89ea33262247e728f7534c9eafd0d</parentId> - <title>Need a real time index along with an Incremental index</title> - <my_contents> search=incr&restrict=abc RJ skrev Hi Jo We have multiple SDs deployed in one cluster and we normally use search= in our queries How do we use it for differentiating both the cluster name and the sd name For example We have two clusters namely incr and realtime each having SDs abc xyz We need a way to query from incr cluster in abc sd Thanks RJ Jo Kristian Bergum wrote On Mon 2009-08-24 at 16:55 +0530 AS wrote On the serving side how do we indicate the cluster to get results from search= /JKB </my_contents> - <lastUpdate>1265031079</lastUpdate> - <pagerank>0</pagerank> - <date>1251115178</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>Need a real time index along with an Incremental index</my_title> - <messageId>3eac9311090c9cf7c34f9d8c897f2c4b</messageId> - <origContents binaryencoding="base64">JnNlYXJjaD1pbmNyJnJlc3RyaWN0PWFiYw0KDQpSYWphdCBKYWluIHNrcmV2Og0KPiBIaSBKbywNCj4NCj4gV2UgaGF2ZSBtdWx0aXBsZSBTRHMgZGVwbG95ZWQgaW4gb25lIGNsdXN0ZXIgYW5kIHdlIG5vcm1hbGx5IHVzZSANCj4gJnNlYXJjaD08c2RuYW1lPiBpbiBvdXIgcXVlcmllcy4gSG93IGRvIHdlIHVzZSBpdCBmb3IgZGlmZmVyZW50aWF0aW5nIA0KPiBib3RoIHRoZSBjbHVzdGVyIG5hbWUgYW5kIHRoZSBzZCBuYW1lPw0KPiBGb3IgZXhhbXBsZSwNCj4gICAgIFdlIGhhdmUgdHdvIGNsdXN0ZXJzIG5hbWVseSAiaW5jciIgYW5kICJyZWFsdGltZSIsIGVhY2ggaGF2aW5nIA0KPiBTRHMsICJhYmMiICYgInh5eiIuIFdlIG5lZWQgYSB3YXkgdG8gcXVlcnkgZnJvbSBpbmNyIGNsdXN0ZXIgaW4gYWJjIHNkLg0KPg0KPiBUaGFua3MhDQo+IFJhamF0IEphaW4NCj4NCj4gSm8gS3Jpc3RpYW4gQmVyZ3VtIHdyb3RlOg0KPj4gT24gTW9uLCAyMDA5LTA4LTI0IGF0IDE2OjU1ICswNTMwLCBBbWl0IFNpbmhhIHdyb3RlOg0KPj4gICANCj4+PiBPbiB0aGUgc2VydmluZyBzaWRlIGhvdyBkbyB3ZSBpbmRpY2F0ZSB0aGUgY2x1c3RlciB0byBnZXQgcmVzdWx0cw0KPj4+IGZyb20uDQo+Pj4gICAgIA0KPj4gJnNlYXJjaD08Y2x1c3Rlcm5hbWU+IA0KPj4NCj4+IC9KS0INCj4+DQo+Pg==</origContents> - <threadUrl>/2009/08/20/partial_update_of_the_entire_document</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::c146d4c9f6153c4573f9a22fd5b19eb3"> - <threadId>5b3a4526d33f45b09f63d43f476f824a</threadId> - <contentsHash>3700f8d3da3a8963f34f11816ffda2cc</contentsHash> - <authors> - <item>bergum</item> - </authors> - <urlPath>/vespa-users/msg/a0c55f0c7669d097e470182bd7d411cf</urlPath> - <contents>On Mon 2009-05-18 at 15:44 +0800 JLB wrote Hi All Am looking for a documentation on how to setup Vespa 1.1.5 I found this link http://vespa/documentation/setup/howtorun.html and it's not available anymore Am just wondering if you have it somewhere If so can you send it to me Thank you Why do you want to use Vespa 1.1.5 It's legacy and not longer supported Please consider using latest stable vespa release http://vespa Best Jo Kristian J plain text document attachment message-footer.txt Vespa Information http://vespa</contents> - <parentId>acca854c8876330b43143c61a1e1f32c</parentId> - <title>Vespa 1.1.5 Documentation</title> - <my_contents>On Mon 2009-05-18 at 15:44 +0800 JLB wrote Hi All Am looking for a documentation on how to setup Vespa 1.1.5 I found this link http://vespa/documentation/setup/howtorun.html and it's not available anymore Am just wondering if you have it somewhere If so can you send it to me Thank you Why do you want to use Vespa 1.1.5 It's legacy and not longer supported Please consider using latest stable vespa release http://vespa Best Jo Kristian J plain text document attachment message-footer.txt Vespa Information http://vesp </my_contents> - <lastUpdate>1265031079</lastUpdate> - <pagerank>0</pagerank> - <date>1242632220</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>Vespa 1.1.5 Documentation</my_title> - <messageId>a0c55f0c7669d097e470182bd7d411cf</messageId> - <origContents binaryencoding="base64">T24gTW9uLCAyMDA5LTA1LTE4IGF0IDE1OjQ0ICswODAwLCBKZXJvbWUgTGVtdWVsIEJhc2Egd3JvdGU6Cj4gSGkgQWxsLAo+IAo+IEFtIGxvb2tpbmcgZm9yIGEgZG9jdW1lbnRhdGlvbiBvbiBob3cgdG8gc2V0dXAgVmVzcGEgMS4xLjUsIEkgZm91bmQgdGhpcyAKPiBsaW5rOiAKPiBodHRwOi8vdmVzcGEudHJvbmRoZWltLmNvcnAueWFob28uY29tLzEuMS41L2RvY3VtZW50YXRpb24vc2V0dXAvaG93dG9ydW4uaHRtbCAKPiBhbmQgaXQncyBub3QgYXZhaWxhYmxlIGFueW1vcmUuIEFtIGp1c3Qgd29uZGVyaW5nIGlmIHlvdSBoYXZlIGl0IAo+IHNvbWV3aGVyZS4gSWYgc28sIGNhbiB5b3Ugc2VuZCBpdCB0byBtZS4gVGhhbmsgeW91LgoKV2h5IGRvIHlvdSB3YW50IHRvIHVzZSBWZXNwYSAxLjEuNT8gSXQncyBsZWdhY3kgYW5kIG5vdCBsb25nZXIKc3VwcG9ydGVkLCBQbGVhc2UgY29uc2lkZXIgdXNpbmcgbGF0ZXN0IHN0YWJsZSB2ZXNwYSByZWxlYXNlLiAKaHR0cDovL3Zlc3BhLmNvcnAueWFob28uY29tLwoKQmVzdCwKSm8gS3Jpc3RpYW4gCgoKCj4gCj4gLUplcm9tZQo+IHBsYWluIHRleHQgZG9jdW1lbnQgYXR0YWNobWVudCAobWVzc2FnZS1mb290ZXIudHh0KQo+IFZlc3BhIEluZm9ybWF0aW9uOgo+ICAgICAgaHR0cDovL3Zlc3BhLmNvcnAueWFob28uY29tLwo+ICAgICAgaHR0cDovL3R3aWtpLmNvcnAueWFob28uY29tL3ZpZXcvVmVzcGEKPg==</origContents> - <threadUrl>/2009/05/18/vespa_1_1_5_documentation</threadUrl> - <level>(NULL)</level> -</document> -<document documenttype="vespacorp" documentid="id:vespacorp:vespacorp::0553f1ea3af33d11aa9ab42496d11f78"> - <threadId>cb34c66221f7188a09e6151062c14e16</threadId> - <contentsHash>fba771ebc63a57f191cf783b0a59298f</contentsHash> - <authors> - <item>peng</item> - </authors> - <urlPath>/vespa-users/msg/079588cac2dfd1117bef476409da724b</urlPath> - <contents>There is no rules to write When you enable libyell_poststemmer if a word is not known by dictionary libyell will try to do plural singular stemming according to its builtin rules Peng Original Message From BK mailto:mail@yhost.com Sent Tuesday July 03 2007 3:10 AM To mail@host.com Subject rule based stemming in vespa Hi I came across this document http://VespaStemming on plural singular stemming I want to try rule based plural singular stemming libyell_poststemmer in vespa Can someone point me to the relevant documents on how to write these rules and use them Thanks B </contents> - <parentId>3be4bf063c46c7e3d336dbfb4c58f6e7</parentId> - <title>Rule based stemming in vespa</title> - <my_contents>There is no rules to write When you enable libyell_poststemmer if a word is not known by dictionary libyell will try to do plural singular stemming according to its builtin rules Peng Original Message From BK mailto:mail@yhost.com Sent Tuesday July 03 2007 3:10 AM To mail@host.com Subject rule based stemming in vespa Hi I came across this document http://LocalVespaStemming on plural singular stemming I want to try rule based plural singular stemming libyell_poststemmer in vespa Can someone point me to the relevant documents on how to write these rules and use them Thanks B </my_contents> - <lastUpdate>1265031079</lastUpdate> - <pagerank>0</pagerank> - <date>1184187769</date> - <headers></headers> - <articleType>email</articleType> - <emailProcessedContents binaryencoding="base64"></emailProcessedContents> - <attachments></attachments> - <documentAbstract></documentAbstract> - <audience>(NULL)</audience> - <docDirId>(NULL)</docDirId> - <visibility>1</visibility> - <headings> - <item></item> - </headings> - <my_title>Rule based stemming in vespa</my_title> - <messageId>079588cac2dfd1117bef476409da724b</messageId> - <origContents binaryencoding="base64">VGhlcmUgaXMgbm8gcnVsZXMgdG8gd3JpdGUuIFdoZW4geW91IGVuYWJsZSBsaWJ5ZWxsX3Bvc3RzdGVtbWVyLCBpZiBhIHdvcmQKaXMgbm90IGtub3duIGJ5IGRpY3Rpb25hcnksIGxpYnllbGwgd2lsbCB0cnkgdG8gZG8gcGx1cmFsLT5zaW5ndWxhciBzdGVtbWluZwphY2NvcmRpbmcgdG8gaXRzIGJ1aWx0aW4gcnVsZXMuCgpQZW5nCgogCgotLS0tLU9yaWdpbmFsIE1lc3NhZ2UtLS0tLQpGcm9tOiBCYWxhamkgS2FubmFuIFttYWlsdG86a2JhbGFqaUB5YWhvby1pbmMuY29tXSAKU2VudDogVHVlc2RheSwgSnVseSAwMywgMjAwNyAzOjEwIEFNClRvOiB2ZXNwYS11c2Vyc0B5YWhvby1pbmMuY29tClN1YmplY3Q6IHJ1bGUgYmFzZWQgc3RlbW1pbmcgaW4gdmVzcGEgCgpIaSwKICBJIGNhbWUgYWNyb3NzIHRoaXMgZG9jdW1lbnQKaHR0cDovL3R3aWtpLmNvcnAueWFob28uY29tL3ZpZXcvTG9jYWxlbmcvTG9jYWxWZXNwYVN0ZW1taW5nIG9uIApwbHVyYWwtPnNpbmd1bGFyIHN0ZW1taW5nLiAgIEkgd2FudCB0byB0cnkgcnVsZSBiYXNlZCBwbHVyYWwtPnNpbmd1bGFyIApzdGVtbWluZyAobGlieWVsbF9wb3N0c3RlbW1lcikgaW4gdmVzcGEuICBDYW4gc29tZW9uZSBwb2ludCBtZSB0byB0aGUKcmVsZXZhbnQgZG9jdW1lbnRzIG9uIGhvdyB0byB3cml0ZSB0aGVzZSBydWxlcyBhbmQgdXNlIHRoZW0uCgpUaGFua3MKQmFsYWpp</origContents> - <threadUrl>/2007/07/03/rule_based_stemming_in_vespa</threadUrl> - <level>(NULL)</level> -</document> -</vespafeed> diff --git a/vespa-http-client/src/test/resources/xml-challenge.xml b/vespa-http-client/src/test/resources/xml-challenge.xml deleted file mode 100644 index 55e368732d7..00000000000 --- a/vespa-http-client/src/test/resources/xml-challenge.xml +++ /dev/null @@ -1,6 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> - <document documenttype="biz" transformver="5681" documentid="id:lsbe:biz::21336977"><attrlist><![CDATA[<other_urls><n>2</n><l><m><url>http://www.facebook.com/pages/City-of-Sunnyvale-California/132586463442411</url><URLTYPE>facebook</URLTYPE></m><m><url>http://www.twitter.com/CityofSunnyvale</url><URLTYPE>twitter</URLTYPE></m></l></other_urls><toc>19,22,36,42,48,74</toc><website><m><src>GRID</src><url>http://www.sunnyvale.ca.gov/</url></m></website><neighbor>Downtown|Sunnyvale Town Center</neighbor><woeId>Zip:12797147;DMA:24701119;State:2347563</woeId><consumersubmit><addbyuser>0</addbyuser></consumersubmit>]]></attrlist></document> - -</vespafeed> diff --git a/vespa-http-client/src/test/resources/xml-challenge2.xml b/vespa-http-client/src/test/resources/xml-challenge2.xml deleted file mode 100644 index 7ed0be68dea..00000000000 --- a/vespa-http-client/src/test/resources/xml-challenge2.xml +++ /dev/null @@ -1,5 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> - <document documenttype="biz" transformver="5681" documentid="id:lsbe:biz::21336977"><version_index>1395987733</version_index><attrlist><other_urls><n>2</n><l><m><url>http://www.facebook.com/pages/City-of-Sunnyvale-California/132586463442411</url><URLTYPE>facebook</URLTYPE></m><m><url>http://www.twitter.com/CityofSunnyvale</url><URLTYPE>twitter</URLTYPE></m></l></other_urls><toc>19,22,36,42,48,74</toc><website><m><src>GRID</src><url>http://www.sunnyvale.ca.gov/</url></m></website><neighbor>Downtown|Sunnyvale Town Center</neighbor><woeId>Zip:12797147;DMA:24701119;State:2347563</woeId><consumersubmit><addbyuser>0</addbyuser></consumersubmit></attrlist><dispycat>96929308:City Hall:1|96927047:Government:1</dispycat><ip_hoo5>0|0|0|0|0|0|0|0</ip_hoo5><ip_popularity_features>0.01401179941|0.0780352748154|0.0|0.00020|0.00008</ip_popularity_features><s1><CTS>96925679 96927047 96929308</CTS><S_CNSUMR_SUBMT>0</S_CNSUMR_SUBMT><DR_PLUS4>7619</DR_PLUS4></s1><ll>-122037613;37371072</ll><stat>1 3 4 6 7 8 11 12 13 14 15 16 33 34 36 41 42 43 44 46 48 49 50 51 57 58 59 60 71 72 73 74 75 76 77 80 81 82 83 85 86 87 131 132 134 137 138 139 140 141 143 144 159 181 183 184 192 193 196 197 198 204</stat><uri>21336977</uri><title>sch | sunnyvalecity | sunnyvalecityhall | hall | city | sunnyvale</title><dtitle>Sunnyvale City Hall</dtitle><ip_hoo2>0|0|0|0|0|0|0|0</ip_hoo2><citystate>Sunnyvale CA</citystate><nhelpreview>0</nhelpreview><ll_long>-122037613</ll_long><ip_custombyte>7|57|103</ip_custombyte><prior_rating>0</prior_rating><ip_hoo4>0|0|0|0|0|0|0|0</ip_hoo4><ip_ycatid2>96929308|96927047</ip_ycatid2><ftitle>20</ftitle><ip_ycat_primary_id>96929308|96927047</ip_ycat_primary_id><ip_eid>NAV=17555658|PSOXSOCIALURL=21336977|INFOUSA=102028107</ip_eid><zip>94086</zip><prior_nreview>0</prior_nreview><ip_customweights>780|33|.09102564102564102|49.732394366197184|121739|3060000|717|1860|716|1860|459|1400|457|1400|114|913|-1|-1|-1|-1|0.0862069|0.12267753</ip_customweights><type>POI</type><enhanced>1</enhanced><ip_normtitle>sunnyvale city hall</ip_normtitle><primary_url>http://www.sunnyvale.ca.gov/</primary_url><ip_hoo1>0|0|0|0|0|0|0|0</ip_hoo1><dma>24701119</dma><city>Sunnyvale</city><ip_catkey>city hall|government</ip_catkey><prior_nrating>0</prior_nrating><lcw_ext><item weight="0">0</item></lcw_ext><pop0>0</pop0><ll_lat>37371072</ll_lat><phone>4087307500</phone><dispfeaturef>17</dispfeaturef><ip_ycat_primary>City Hall|Government</ip_ycat_primary><ip_hoo6>0|0|0|0|0|0|0|0</ip_hoo6><ip_ycat_primary_synonyms>county government|marriage licenses|us government|government-office|government relations firms|government agencies|government relations|marriage license|city government departments|city government|federal government|usgovernment|council of governments|governmentoffices|government offices|government officials|government-contract consultants|governments offices</ip_ycat_primary_synonyms><ip_dcat>City Hall|Government</ip_dcat><listing_status>103</listing_status><lcw_pcat><item weight="7071">96929308</item><item weight="7071">96927047</item></lcw_pcat><pop_keyword_portion><item weight="2">3053931</item><item weight="1">99043162</item><item weight="57">-734328434</item><item weight="6">-1164754482</item><item weight="33">-1164681749</item><item weight="10000">-1491073460</item><item weight="5000">1098856921</item><item weight="5000">90458857</item><item weight="10000">-1956199191</item><item weight="10000">-2015626891</item><item weight="10000">38487508</item><item weight="1428">-1384211533</item><item weight="588">-995864441</item><item weight="6666">-350418064</item><item weight="112">-1493944221</item></pop_keyword_portion><stat2>2 3 4 5 7 8 10 11 21 24 31 42 44 45 46 47 48 50 51 53 54 56 57 69 92 112 121 123 128 129 134 154 195 205 232 238 242</stat2><desc> </desc><ip_rating>0</ip_rating><lcw><item weight="5745">96927047</item><item weight="8186">96929308</item></lcw><pop_keyword_certainty><item weight="5425">3053931</item><item weight="5344">99043162</item><item weight="7105">-734328434</item><item weight="5140">-1164754482</item><item weight="6766">-1164681749</item><item weight="10000">-1491073460</item><item weight="9587">1098856921</item><item weight="9587">90458857</item><item weight="10000">-1956199191</item><item weight="10000">-2015626891</item><item weight="10000">38487508</item><item weight="8842">-1384211533</item><item weight="9120">-995864441</item><item weight="9621">-350418064</item><item weight="7429">-1493944221</item></pop_keyword_certainty><q>9</q><isactive>1</isactive><ip_ycat2gc>96929308|96927047</ip_ycat2gc><ip_hoo0>0|0|0|0|0|0|0|0</ip_hoo0><language>en</language><nreview>0</nreview><ip_pycatnames>Government & Community</ip_pycatnames><ip_catkey_click>marriage license</ip_catkey_click><country_code>us</country_code><state>CA</state><ip_hoo3>0|0|0|0|0|0|0|0</ip_hoo3><version_pub>1346889600</version_pub><ip_ycat_primary_cp_desc>CITY GOVERNMENT-EXECUTIVE OFFICES</ip_ycat_primary_cp_desc><crossst>2|All America Way|Charles St</crossst><ip_dcatkey>us government|government relations firms|marriage license|federal government|council of governments|governmentoffices|government officials|government-contract consultants|governments offices|county government|marriage licenses|government-office|government agencies|government relations|city government departments|city government|usgovernment|government offices</ip_dcatkey><dispambiancef>7</dispambiancef><ip_ycatid1>96925679</ip_ycatid1><paid_listing_status>0</paid_listing_status><lcw_norm>0.5640622</lcw_norm><webkeyword>city sunnyvale twitter jobs news contact us about codes policies charter municipal code council policy general plan maps directions map hall external link library public safety parks golf courses swimming pools tennis center transportation area resources community resource guide frequently requested english spanish budget documents data demographics business economic profile learn new resident information sheet this site privacy what hot topics plastic bag ban amendment ballot measures elections firearms retail study issue review committee medical marijuana dispensary horizon downtown redevelopment sustainability consolidation lute update manager updates meetings agendas next meeting february boards commissions latest publications fall activity winter quarterly report current job openings bid on projects open bids around cited as american top potential upcoming events issues workshop feb offices are closed more event calendar government agenda watch making presentation at arts bicycle pedestrian board trustees building appeals heritage preservation housing human recreation personnel planning other agencies county santa clara state california clerk page campaign ethics departments attorney development environmental nova workforce finance technology works living get card search catalog manage your account getting involved volunteering neighborhood associations classes activities out now playing performing shopping dining in centers columbia senior guides unemployed residents infrastructure traffic trees street maintenance garbage recycling smart station extra tags affordable assistance water supply pollution control plant police fire emergency preparedness alarms enforcement animal crime prevention records recruiting permits special residential non one stop permit checks fees encroachment tree removal doing starting facts figures bidding process shop auto row available commercial properties licenses reports links division newsroom recent releases apple occupy president day holiday closures meet local author francisco jimenez reads muslim door arrest made fatal hit run le jazz copper wire theft hits television broadcast schedule social media follow want apply license block party sign up service call tee times facility reservations compost pay my utility bill fines violation graffiti pothole web dispose old medication waste hazardous trash find department places dine volunteer access file claim help improve list something missing from that would make it even useful let know we welcome suggestions friday share comments tuesday strategic featured telephone scam claims be computer has received complaints apparent phone which caller identifies himself representing victim windows system transmitting bad does not these kind calls have if receive type desk officer gary announced will join ranks high tech businesses located square foot town office mathilda mckinley avenues important step forward said entire benefit see yet another large gain project read planned by pg along caribbean drive remove number mostly eucalyptus measure underground gas pipeline young replacement planted median controls during work or go amp presidents facilities including monday observance all parking regulations enforced except where signs specifically exempt holidays collection continue normal posted edition gives tip leading burglars vargas elementary school partnership marathon club kids led streetlight free healthy toddler workshops electronic commission change please note dates do typical due construction chambers scheduled wednesday artists applications hands festival participate may downloading click here download application acclaimed quartet performs gypsy la reinhardt version internationally recognized san perform valentine weekend concert style theatre saturday silicon valley diverse religions cultures been celebrated each year reading choices continues select provocative relevant theme off air channel broadcasts equipment upgrade allowing compliant standards begin broadcasting again no down time so missed vehicle versus accident occurred intersection sequoia reed avenue january driver black struck benjamin lin did major investigation team developed significant leads case identification seizure warrant was issued popular register november networks google divider final eir sales hazard mitigation subscribe notifications feed icon olive ave ca logo can check stay touch used maintained communications questions fine print terms use</webkeyword><ip_provider>NAV|PSOXSOCIALURL|INFOUSA</ip_provider><nrating>0</nrating><ratingfgc>0</ratingfgc><ip_ycat2>96925679</ip_ycat2><spaid>N</spaid><ip_cat>GOVERNMENTOFFICES|OFFICES|HALL|GOVERNMENT|DEPARTMENTS|GOVERNMENTS|CONSULTANTS|OFFICIALS|RELATIONS|USGOVERNMENT|OFFICE|CONTRACT|COUNCIL|CITY|COMMUNITY|FIRMS|AGENCIES|COUNTY</ip_cat><ip_neighborhood>downtownsunnyvaleca|sunnyvaletowncentersunnyvaleca</ip_neighborhood><addrhash>068B74475DB0D415</addrhash><webtext>City of Sunnyvale: Home * Twitter * | * Jobs * | * News * | * eNotify * | * RSS * | * Contact Us Home About The City Codes and Policies City Charter Municipal Code Council Policy General Plan Maps and Directions Map of Sunnyvale City Hall external link Library Public Safety City Parks Golf Courses Swimming Pools Tennis Center Public Transportation Area Resources Community Resource Guide to Frequently Requested Services: English (pdf) | Spanish (pdf) City Budget Budget Documents Data and Demographics Business Demographics Economic Profile (pdf) Learn about Sunnyvale New Resident Guide City Council Information Sheet About This Site About the Site Site Map Privacy Policy Contact Us Whats New Hot Topics Plastic Bag Ban Charter Amendment Ballot Measures City Council Elections Firearms Retail Study Issue Charter Review Committee Medical Marijuana Dispensary Study Horizon 2035 Downtown Redevelopment Onizuka / BRAC Sustainability General Plan Consolidation LUTE Update City Managers Updates Meetings and Agendas Next Council Meeting: February 7 Council Meetings Boards and Commissions Latest Publications Fall Activity Guide Winter 2012 Quarterly Report 2010 New Resident Guide Jobs Current Job Openings Bid on Projects Current Open Bids Around the City Sunnyvale Cited As American City with Top Economic Potential Upcoming Events February 3 - Council Budget/Study Issues Workshop Feb 20 - City Offices are closed More Events on the Community Event Calendar Government City Council About Council Current Council Agenda Councilmembers Council Meeting Agendas Watch Council Meetings Online Making a Presentation at Council Boards and Commissions About Boards and Commissions Current Openings Arts Bicycle and Pedestrian Board of Library Trustees Building Code Appeals Heritage Preservation Housing and Human Services Parks and Recreation Personnel Planning Sustainability Study Issues 2012 Study Issues 2011 Study Issues 2010 Study Issues 2009 Study Issues Other Agencies County of Santa Clara external link State of California external link Codes and Policies City Charter Municipal Code Council Policy General Plan Elections City Clerk Elections Page Campaign Ethics Guide Departments City Attorney City Manager Community Development Community Services Environmental Services NOVA Workforce Services Finance Human Resources Information Technology Public Safety Public Works Sunnyvale Public Library Living Library Library Home Page Get a Library Card Search the Library Catalog Manage Your Library Account Getting Involved Volunteering Neighborhood Associations Recreation Golf Tennis Parks Classes and Activities Swimming Out and About Now Playing at the Performing Arts Center Shopping and Dining in Sunnyvale Community Event Calendar Community Centers Columbia Neighborhood Center Sunnyvale Community Center Sunnyvale Senior Center Resource Guides Resources for Unemployed Residents Community Resource Guide to Frequently Requested Services: English (pdf) | Spanish (pdf) Services City Infrastructure Traffic and Transportation Trees Street Maintenance Garbage and Recycling SMaRT Station Garbage and Recycling Services Extra Garbage Tags Housing Affordable Housing and Community Assistance Water Water Supply Water Pollution Control Plant (WPCP) Public Safety Police Fire Emergency Preparedness Alarms Code Enforcement Animal Control Crime Prevention Public Safety Records Public Safety Recruiting Permits Special Event Permits Residential Permits Non-residential Permits E-Onestop One-Stop Permit Center Permits, Plan Checks and Fees Encroachment Permits Tree Removal Permits Doing Business Sunnyvale for Business Starting a Business in Sunnyvale Facts and Figures Doing Business in Sunnyvale Economic Development Downtown Development Bid on Sunnyvale Projects The Bidding Process Current Open Bids Shop Sunnyvale Sunnyvale Auto Row Business Resources Available Commercial Properties E-One Stop Permit Center Business Licenses Business News and Reports Business Links Building Division Planning Division Newsroom Recent News Releases Apple to Occupy New Downtown Sunnyvale Offices Presidents Day Holiday Closures Meet Sunnyvale Local Author Francisco Jimenez Sunnyvale Reads the Muslim Next Door Arrest Made in Fatal Sunnyvale Hit-And-Run Le Jazz Hot in Sunnyvale February 11 Copper Wire Theft Hits City of Sunnyvale More News Releases Publications Quarterly Report Activity Guide Senior Activity Guide New Resident Guide Campaign Ethics Guide On Television KSUN Broadcast Schedule Social Media Follow Us on Twitter City of Sunnyvale Facebook Page I Want To . Apply For: Job Openings Boards and Commissions Business License Permits Animal License Special Event Permit Block Party Sign Up For: Garbage Service Recycling Service On-Call Garbage/Recycling Service Recreation Classes Golf Tee Times Facility Reservations Compost Workshop Pay For: My Utility Bill Library Fines Report: Code Violation Graffiti Pothole Public Safety Issue Traffic Issue General Issue Web Site Issue Dispose of: Old Medication E-Waste Other Hazardous Waste Extra Trash Find: Building Permits City Hall external link City Parks Sunnyvale Community Center Columbia Neighborhood Center Sunnyvale Senior Center Sunnyvale Public Library Department of Public Safety Places to Shop and Dine Watch: Council Meetings Online Other: Volunteer in the City Access Public Records File a Claim Help Improve This List Is something missing from this list that would make it an even more useful resource? Let us know! We welcome your suggestions. Contact us online. Friday, February 10, 2012 Share Your Comments Next Council Meeting * Tuesday, February 7, 2012 * Council Agendas * Watch the Meeting Online * Watch the Meeting on KSUN-15 * About Council Upcoming Events * February 10 - Strategic Planning Workshop * February 20 - City Holiday - Offices closed * Community Events Calendar Recent News Releases Apple to Occupy New Downtown Sunnyvale Offices Presidents Day Holiday Closures Meet Sunnyvale Local Author Francisco Jimenez Sunnyvale Reads the Muslim Next Door Arrest Made in Fatal Sunnyvale Hit-And-Run Le Jazz Hot in Sunnyvale February 11 More News Releases Featured City News Telephone Scam Claims to be from Sunnyvale Computer Department The City has received complaints of an apparent phone scam in which the caller identifies himself as representing the Sunnyvale Computer Department. The caller claims the victims computers Windows system is transmitting bad data. City of Sunnyvale does not make these kind of calls to the public and does not have a Computer Department. If you receive this type of scam call, report the call to a Public Safety Desk Officer at (408) 730-7110.Learn More Telephone Scam Claims to be from "Sunnyvale Computer Department" Apple to occupy new downtown Sunnyvale offices Sunnyvale City Manager Gary Luebbers has announced that Apple will join the ranks of high-tech businesses located in downtown Sunnyvale. Apple will occupy the new 156,000 square-foot Town Center Office building at Mathilda and McKinley avenues. ?This is an important step forward,? said Luebbers. Our entire community will benefit as we see yet another large gain in the Town Center redevelopment project.? Read more Learn More Apple to occupy new downtown Sunnyvale offices Tree Removal Planned by PG&E along Caribbean Drive PG&E will remove a number of large trees ? mostly eucalyptus ? along Caribbean Drive, February 6 ? 15. This is a safety measure for the underground gas pipeline. Young replacement trees will planted in the median; watch for traffic controls during this work. For more information, call PG&E at (800) 743-5000 or go to http://pge.com/myhome/customerservice/Learn More Tree Removal Planned by PG&amp;E along Caribbean Drive City Closed for Presidents Day Sunnyvale City offices and facilities, including the Sunnyvale Public Library, Community Center, Senior Center and Columbia Neighborhood Center will be closed Monday, February 20, in observance of the Presidents Day holiday. All traffic and parking regulations will be enforced, except for parking where signs specifically exempt holidays. Trash collection will continue on a normal schedule. Learn More City Closed for Presidents Day Latest City Managers Blog Posted In the latest edition of the City Managers Blog, Gary Luebbers gives an update on the 2012 Study Issues, a public tip leading to the arrest of local burglars, the Vargas Elementary School partnership with Public Safety for a Marathon Club for kids, latest street updates and the LED Streetlight project, free healthy toddler workshops at the Library, a new electronic Job Board from NOVA, and more Learn More Latest City Managers Blog Posted Planning Commission February Meeting Schedule Change PLEASE NOTE: The February Planning Commission meeting dates do not follow the typical schedule due to construction in the Council Chambers. The February Planning Commission Meetings are scheduled for: * MONDAY, FEBRUARY 6, 2012 * WEDNESDAY, FEBRUARY 29, 2012 Learn More Planning Commission February Meeting Schedule Change Artists Applications for 2012 Hands on the Arts Festival Now Posted Applications for artists to participate in the 2012 Hands on the Arts Festival in Sunnyvale on May 19 are now available for downloading. Click here to read more and download the application Learn More Artists Applications for 2012 Hands on the Arts Festival Now Posted Le Jazz Hot in Sunnyvale February 11 Acclaimed quartet performs Gypsy jazz la Django Reinhardt Le Jazz Hot, the quartet version of internationally recognized Le Hot Club of San Francisco, will perform a Valentine?s weekend concert of Django Reinhardt-style Gypsy jazz at Sunnyvale Theatre on Saturday, February 11, at 8 p.m. Read more Learn More Le Jazz Hot in Sunnyvale February 11 Sunnyvale Reads The Muslim Next Door Silicon Valley?s diverse religions and cultures have been celebrated each year by reading choices from Silicon Valley Reads, which continues to select provocative topics relevant to Santa Clara County. The theme for Silicon Valley Reads 2012 is ?Muslim and American.? Read more Learn More Sunnyvale Reads "The Muslim Next Door" KSUN-15 Off Air February 8-27 for Ugrades The Citys public access channel KSUN-15, which broadcasts Council and Planning Commission meetings, will be off-air starting February 8 to get an equipment upgrade, allowing the system to be compliant with current broadcast standards. KSUN will begin broadcasting again on February 27. No meetings are planned during this down time so no broadcasts will be missed. Learn More KSUN-15 Off Air February 8-27 for Ugrades Arrest Made in Fatal Hit-and-Run in Sunnyvale A fatal vehicle versus pedestrian accident occurred at the intersection of Sequoia Drive and Reed Avenue in Sunnyvale on January 5. The driver of a black SUV struck 72-year-old Benjamin Lin and did not stop. The Major Accident Investigation Team (MAIT) developed significant leads in the case which led to the identification of the driver and seizure of the vehicle. On January 19, a $500,000 arrest warrant was issued. Read more Learn More Arrest Made in Fatal Hit-and-Run in Sunnyvale Popular Links * Jobs * Sunnyvale Public Library * Register for Classes and Activities * Recycling and Garbage Information * Volunteer * Maps and Directions * Pay Your Utility Bill Online * November 2011 Library Events Calendar in PDF Pay Your Utility Bill Online Utility Bill Online Pay Doing Business * Bid on City Projects * Economic Development * Business Licenses * Apply for Permits Online Social Networks * +1 us on Google+ * City of Sunnyvale Twitter Account * City of Sunnyvale Facebook Page divider Hot Topics * Tennis Center * Plastic Bag Ban Final EIR * Firearms Sales Study Issue * Sunnyvale Works! * Downtown Development * Sustainability * Onizuka / BRAC * Local Hazard Mitigation Plan (LMPH) Subscribe * e-Notifications * City Managers Blog * RSS Feed * RSS Feed icon City of Sunnyvale (408) 730-7500 * Sunnyvale City Hall * 456 W. Olive Ave. * Sunnyvale, CA 94086 * TDD (408) 730-7501 * Map and Directions * City of Sunnyvale Logo Cant Find It? * Or check out the SITE MAP! Stay In Touch * Contact Us * Follow us on Twitter Frequently-Used Links * Jobs with the City * Library * Garbage and Recycling * Downtown Redevelopment * Economic Development * Register for Classes and Activities About the City * Welcome to Sunnyvale * City Charter and Policies * City Council * City Departments * New Resident Guide About the Web Site The City of Sunnyvale Web Site is maintained by the Sunnyvale Communications Office and the Department of Information Technology. Questions? Contact Us. The Fine Print * Privacy Policy * Terms of Use 2010 City of Sunnyvale</webtext><ip_keyword>[ ca ]</ip_keyword><addr>456 W Olive Ave</addr></document> -</vespafeed> diff --git a/vespa-http-client/src/test/resources/xml-challenge3.xml b/vespa-http-client/src/test/resources/xml-challenge3.xml deleted file mode 100644 index be0f789f870..00000000000 --- a/vespa-http-client/src/test/resources/xml-challenge3.xml +++ /dev/null @@ -1,4 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<vespafeed> - <document documenttype="biz" transformver="5681" documentid="id:lsbe:biz::21336977"><attrlist>''</attrlist></document></vespafeed> |