summaryrefslogtreecommitdiffstats
path: root/vespa-http-client/src/test/java
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@gmail.com>2022-06-05 13:04:17 +0200
committergjoranv <gv@verizonmedia.com>2022-06-08 11:45:30 +0200
commit8647f18a80c40c3fe98f7f68e0330683786eb50d (patch)
tree9cb2a4bb9658f40de10b6ded892b54f928ed6e6f /vespa-http-client/src/test/java
parenteabb1d3195a1bd6d34def333854ad713bd6ac886 (diff)
Remove vespa-http-client on Vespa 8
Diffstat (limited to 'vespa-http-client/src/test/java')
-rw-r--r--vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java83
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java87
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java55
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java39
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java92
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java111
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java23
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java62
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java38
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java73
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java62
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java54
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java40
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java240
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java125
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java76
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java254
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java28
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java355
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java134
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java52
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java89
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java56
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java175
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java63
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java294
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java438
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java417
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java210
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java114
-rw-r--r--vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java270
31 files changed, 0 insertions, 4209 deletions
diff --git a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java b/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java
deleted file mode 100644
index 2ca2ea4d14a..00000000000
--- a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-import com.yahoo.vespa.http.client.FeedClient;
-import com.yahoo.vespa.http.client.FeedClientFactory;
-import com.yahoo.vespa.http.client.Result;
-import com.yahoo.vespa.http.client.Server;
-import com.yahoo.vespa.http.client.config.Cluster;
-import com.yahoo.vespa.http.client.config.Endpoint;
-import com.yahoo.vespa.http.client.config.FeedParams;
-import com.yahoo.vespa.http.client.config.SessionParams;
-import com.yahoo.vespa.http.client.handlers.V3MockParsingRequestHandler;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Unit test that test documentation code.
- *
- * @author dybis
- */
-public class ExampleUsageFeedClientTest {
-
- @Test
- public void testExampleCode() {
- Server serverA =
- new Server(new V3MockParsingRequestHandler(200, V3MockParsingRequestHandler.Scenario.ALL_OK), 0);
- Server serverB =
- new Server(new V3MockParsingRequestHandler(200, V3MockParsingRequestHandler.Scenario.ALL_OK), 0);
-
- exampleCode("localhost", serverA.getPort(), "localhost", serverB.getPort());
- serverA.close();
- serverB.close();
- }
-
- private static CharSequence generateDocument(String docId) {
- // Just a dummy example of an update document operation.
- return "{\"update\": \""+ docId + "\","
- + " \"fields\": { \"actualMapStringToArrayOfInt\": {"
- + " \"assign\": ["
- + "{ \"key\": \"fooKey\", \"value\": [ 2,1, 3] }"
- + "]}}}";
- }
-
- // Example usage of FeedClient
- public static void exampleCode(String hostNameA, int portServerA, String hostNameB, int portServerB) {
- boolean useSsl = false;
- final SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameA, portServerA, useSsl)).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameB, portServerB, useSsl)).build())
- .setFeedParams(new FeedParams.Builder()
- .setDataFormat(FeedParams.DataFormat.JSON_UTF8)
- .build())
- .build();
-
- AtomicInteger resultsReceived = new AtomicInteger(0);
- AtomicInteger errorsReceived = new AtomicInteger(0);
-
- FeedClient feedClient = FeedClientFactory.create(sessionParams, new FeedClient.ResultCallback() {
- @Override
- public void onCompletion(String docId, Result documentResult) {
- resultsReceived.incrementAndGet();
- if (! documentResult.getContext().equals(docId)) {
- System.err.println("Context does not work as expected.");
- errorsReceived.incrementAndGet();
- }
- if (!documentResult.isSuccess()) {
- System.err.println("Problems with docID " + docId + ":" + documentResult.toString());
- errorsReceived.incrementAndGet();
- }
- }
- });
- int sentCounter = 0;
- List<String> docIds = Arrays.asList("1", "2", "3", "4");
- for (final String docId : docIds) {
- CharSequence docData = generateDocument(docId);
- feedClient.stream(docId, docData, docId);
- sentCounter++;
- }
- feedClient.close();
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java
deleted file mode 100644
index ca956110a34..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client;
-
-import com.yahoo.vespa.http.client.config.Cluster;
-import com.yahoo.vespa.http.client.config.ConnectionParams;
-import com.yahoo.vespa.http.client.config.Endpoint;
-import com.yahoo.vespa.http.client.config.SessionParams;
-import com.yahoo.vespa.http.client.core.api.FeedClientImpl;
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.time.Clock;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Tests for the API, using dryrun option to mock gateway.
- *
- * @author dybis
- */
-public class FeedClientTest {
-
- private final static String DOCID = "doc_id";
-
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder()
- .addEndpoint(Endpoint.create("localhost"))
- .build())
- .setConnectionParams(new ConnectionParams.Builder()
- .setDryRun(true)
- .build())
- .build();
- final AtomicInteger resultsReceived = new AtomicInteger(0);
-
- FeedClient.ResultCallback resultCallback = (docId, documentResult) -> {
- assertTrue(documentResult.isSuccess());
- assertEquals(DOCID, docId);
- resultsReceived.incrementAndGet();
- };
-
- FeedClient feedClient = new FeedClientImpl(sessionParams, resultCallback, FeedClientFactory.createTimeoutExecutor(), Clock.systemUTC());
-
- @Test
- public void testStreamAndClose() {
- feedClient.stream(DOCID, "blob");
- feedClient.close();
- assertEquals(1, resultsReceived.get());
- }
-
- @Test
- public void testGetStatsAsJson() throws Exception {
- feedClient.stream(DOCID, "blob");
- while (resultsReceived.get() == 0) {Thread.sleep(3); }
- String stats = feedClient.getStatsAsJson();
- assertTrue(stats.contains("\"dryRun\":true"));
- feedClient.close();
- }
-
- @Test
- public void testFeedJson() {
- InputStream stream = new ByteArrayInputStream((String.format("[{\"remove\": \"%s\"}]", DOCID)
- .getBytes(StandardCharsets.UTF_8)));
- AtomicInteger docCounter = new AtomicInteger(0);
- FeedClient.feedJson(stream, feedClient, docCounter);
- assertEquals(1, docCounter.get());
- feedClient.close();
- assertEquals(1, resultsReceived.get());
- }
-
- @Test
- public void testFeedXml() {
- InputStream stream = new ByteArrayInputStream((String.format(
- "<document documenttype=\"music\" documentid=\"%s\">\n</document>\n", DOCID)
- .getBytes(StandardCharsets.UTF_8)));
- AtomicInteger docCounter = new AtomicInteger(0);
- FeedClient.feedXml(stream, feedClient, docCounter);
- assertEquals(1, docCounter.get());
- feedClient.close();
- assertEquals(1, resultsReceived.get());
- }
-
-}
-
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java
deleted file mode 100644
index 72edba7adb3..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client;
-
-import java.time.Clock;
-import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-import java.time.temporal.TemporalAmount;
-
-/**
- * A clock which initially has the time of its creation but can only be advanced by calling advance
- *
- * @author bratseth
- */
-public class ManualClock extends Clock {
-
- private Instant currentTime = Instant.now();
-
- public ManualClock() {}
-
- public ManualClock(String utcIsoTime) {
- this(at(utcIsoTime));
- }
-
- public ManualClock(Instant currentTime) {
- this.currentTime = currentTime;
- }
-
- public void advance(TemporalAmount temporal) {
- currentTime = currentTime.plus(temporal);
- }
-
- public void setInstant(Instant time) {
- currentTime = time;
- }
-
- @Override
- public Instant instant() { return currentTime; }
-
- @Override
- public ZoneId getZone() { return null; }
-
- @Override
- public Clock withZone(ZoneId zone) { return null; }
-
- @Override
- public long millis() { return currentTime.toEpochMilli(); }
-
- public static Instant at(String utcIsoTime) {
- return LocalDateTime.parse(utcIsoTime, DateTimeFormatter.ISO_DATE_TIME).atZone(ZoneOffset.UTC).toInstant();
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java
deleted file mode 100644
index 078606274d6..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java
+++ /dev/null
@@ -1,39 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client;
-
-import org.eclipse.jetty.server.ServerConnector;
-import org.eclipse.jetty.server.handler.AbstractHandler;
-
-/**
- * @author Einar M R Rosenvinge
- */
-public final class Server implements AutoCloseable {
-
- private final org.eclipse.jetty.server.Server server;
-
- public Server(AbstractHandler handler, int port) {
- this.server = new org.eclipse.jetty.server.Server(port);
- server.setHandler(handler);
- try {
- server.start();
- assert(server.isStarted());
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void close() throws RuntimeException {
- try {
- server.stop();
- } catch (RuntimeException e) {
- throw e;
- } catch (Exception e) {
- throw new RuntimeException("jetty server.stop() failed", e);
- }
- }
-
- public int getPort() {
- return ((ServerConnector)server.getConnectors()[0]).getLocalPort();
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java
deleted file mode 100644
index 5f4a0fdec9a..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java
+++ /dev/null
@@ -1,92 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client;
-
-import org.junit.Test;
-
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.hamcrest.CoreMatchers.containsString;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class SimpleLoggerResultCallbackTest {
- @Test
- public void testAverageCalculation() {
- SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false);
- Instant now = Instant.now();
- logger.newSamplingPeriod(now);
- Result result = mock(Result.class);
- when(result.isSuccess()).thenReturn(true);
- // 3 documents in 0.2 secs --> 15 docs/sec
- logger.onCompletion("1", result);
- logger.onCompletion("1", result);
- logger.onCompletion("1", result);
- double rate = logger.newSamplingPeriod(now.plusMillis(200)).rate;
- assertEquals(rate, 15., 0.1 /* delta */);
- }
-
- @Test
- public void testAverageCalculationExteremeValues() {
- SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false);
- Instant now = Instant.now();
- logger.newSamplingPeriod(now);
- // 0 duration, 0 documents
- double rate = logger.newSamplingPeriod(now).rate;
- assertEquals(rate, 0, 0.1 /* delta */);
- }
-
- @Test
- public void testOutput() {
- SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false);
- Instant now = Instant.now();
- logger.newSamplingPeriod(now);
- Result result = mock(Result.class);
- when(result.isSuccess()).thenReturn(true);
- // 3 documents in 0.2 secs --> 15 docs/sec
- logger.onCompletion("1", result);
- logger.onCompletion("1", result);
- logger.onCompletion("1", result);
- double rate = logger.newSamplingPeriod(now.plusMillis(200)).rate;
- assertEquals(rate, 15., 0.1 /* delta */);
- }
-
- private void verifyPrintout(boolean ignoreConditionNotMet) {
- ArrayList<String> outputList = new ArrayList<>();
-
- SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(30), 0, ignoreConditionNotMet) {
- @Override
- protected void println(String output) {
- outputList.add(output);
- }
- @Override
- protected DocumentRate newSamplingPeriod(Instant now) {
- return new DocumentRate(19999999.2342342366664);
- }
- };
- // 2 success, 1 failure
- Result result = mock(Result.class);
- when(result.isSuccess()).thenReturn(true);
- when(result.isSuccessOrConditionNotMet()).thenReturn(true);
- logger.onCompletion("1", result);
- logger.onCompletion("1", result);
- when(result.isSuccess()).thenReturn(false);
- when(result.isSuccessOrConditionNotMet()).thenReturn(false);
- when(result.toString()).thenReturn("fooError");
- logger.onCompletion("1", result);
- logger.printProgress();
- assertThat(outputList.toString(),
- containsString("Result received: 3 (1 failed so far, 30 sent, success rate 19999999.23 docs/sec)."));
- assertThat(outputList.toString(), containsString("Failure: fooError"));
- }
-
- @Test
- public void testPrintout() {
- verifyPrintout(false);
- verifyPrintout(true);
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java
deleted file mode 100644
index 53b2b236d50..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java
+++ /dev/null
@@ -1,111 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client;
-
-import com.yahoo.vespa.http.client.config.Cluster;
-import com.yahoo.vespa.http.client.config.ConnectionParams;
-import com.yahoo.vespa.http.client.config.Endpoint;
-import com.yahoo.vespa.http.client.config.FeedParams;
-import com.yahoo.vespa.http.client.config.SessionParams;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.yahoo.vespa.http.client.SyncFeedClient.SyncOperation;
-import com.yahoo.vespa.http.client.SyncFeedClient.SyncResult;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertNull;
-
-/**
- * Tests the sync wrapper to the feed client
- *
- * @author bratseth
- */
-public class SyncFeedClientTest {
-
- @Test
- public void testFeedJson() {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder()
- .addEndpoint(Endpoint.create("localhost"))
- .build())
- .setConnectionParams(new ConnectionParams.Builder()
- .setDryRun(true)
- .build())
- .build();
- SyncFeedClient feedClient = new SyncFeedClient(sessionParams);
-
- assertFeedSuccessful(feedClient);
- assertFeedSuccessful(feedClient); // ensure the client can be reused
- feedClient.close();
- }
-
- private void assertFeedSuccessful(SyncFeedClient feedClient) {
- List<SyncOperation> operations = new ArrayList<>();
-
- operations.add(new SyncOperation("id::test::1",
- "{" +
- " \"put\": \"id::test::1\"," +
- " \"fields\": {" +
- " \"title\": \"Title 1\"" +
- " }" +
- "}"));
- operations.add(new SyncOperation("id::test::2",
- "{" +
- " \"put\": \"id::test::2\"," +
- " \"fields\": {" +
- " \"title\": \"Title 2\"" +
- " }" +
- "}"));
- operations.add(new SyncOperation("id::test::3",
- "{" +
- " \"put\": \"id::test::3\"," +
- " \"fields\": {" +
- " \"title\": \"Title 3\"" +
- " }" +
- "}"));
- operations.add(new SyncOperation("id::test::3", // Another operation for the same document
- "{" +
- " \"put\": \"id::test::3\"," +
- " \"fields\": {" +
- " \"title\": \"Title 4\"" +
- " }" +
- "}"));
- operations.add(new SyncOperation("id::test::4",
- "{" +
- " \"put\": \"id::test::4\"," +
- " \"fields\": {" +
- " \"title\": \"Title 4\"" +
- " }" +
- "}", "opId_4", null));
- operations.add(new SyncOperation("id::test::4", // Another operation for the same document
- "{" +
- " \"put\": \"id::test::4\"," +
- " \"fields\": {" +
- " \"title\": \"Title 44\"" +
- " }" +
- "}", "opId_44", null));
-
- SyncResult result = feedClient.stream(operations);
-
- assertTrue(result.isSuccess());
- assertEquals(6, result.results().size());
- assertNull(result.exception());
- assertEquals("id::test::1", result.results().get(0).getDocumentId());
- assertEquals("id::test::2", result.results().get(1).getDocumentId());
- assertEquals("id::test::3", result.results().get(2).getDocumentId());
- assertEquals("id::test::3", result.results().get(3).getDocumentId());
- assertEquals("id::test::4", result.results().get(4).getDocumentId());
- assertEquals("id::test::4", result.results().get(5).getDocumentId());
- assertEquals("opId_4", result.results().get(4).getOperationId());
- assertEquals("opId_44", result.results().get(5).getOperationId());
- assertTrue(result.results().get(4).getDocumentDataAsCharSequence().toString().contains("\"Title 4\""));
- assertTrue(result.results().get(5).getDocumentDataAsCharSequence().toString().contains("\"Title 44\""));
-
- result.results().forEach(r -> assertNotNull(r.getOperationId()));
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java
deleted file mode 100644
index 67b3fc7653d..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client;
-
-/**
-* @author Einar M R Rosenvinge
-*/
-public class TestDocument {
- private final String documentId;
- private final byte[] contents;
-
- TestDocument(String documentId, byte[] contents) {
- this.documentId = documentId;
- this.contents = contents;
- }
-
- public String getDocumentId() {
- return documentId;
- }
-
- public byte[] getContents() {
- return contents;
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java
deleted file mode 100644
index b27fbba3e96..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java
+++ /dev/null
@@ -1,62 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.zip.GZIPInputStream;
-
-import static org.junit.Assert.assertNull;
-
-/**
- * @author Einar M R Rosenvinge
- */
-@SuppressWarnings("deprecation")
-public class TestUtils {
-
- public static void writeDocuments(Session session, List<TestDocument> documents) throws IOException {
- for (TestDocument document : documents) {
- writeDocument(session, document);
- }
- }
-
- public static void writeDocument(Session session, TestDocument document) throws IOException {
- OutputStream operation = session.stream(document.getDocumentId());
- operation.write(document.getContents());
- operation.close();
- }
-
- public static Map<String, Result> getResults(Session session, int num) throws InterruptedException {
- Map<String, Result> results = new HashMap<>();
- for (int i = 0; i < num; i++) {
- Result r = session.results().poll(120, TimeUnit.SECONDS);
- if (r == null) {
- String extraInfo = "";
- extraInfo = "stats=" + session.getStatsAsJson();
- throw new IllegalStateException("Did not receive result within timeout. (" + extraInfo + ") " +
- "Results received: " + results.values());
- }
- results.put(r.getDocumentId(), r);
- }
- assertNull(session.results().poll(100, TimeUnit.MILLISECONDS));
- return results;
- }
-
- public static String zipStreamToString(InputStream inputStream) throws IOException {
- GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream);
- final StringBuilder rawContent = new StringBuilder();
- while (true) {
- int x = gzipInputStream.read();
- if (x < 0) {
- break;
- }
- rawContent.append((char) x);
- }
- return rawContent.toString();
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java
deleted file mode 100644
index f4c12974f6d..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.config;
-
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
-/**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- * @since 5.1.22
- */
-public class ClusterTest {
-
- @Test
- public void testSimple() {
- Cluster cluster = new Cluster.Builder().build();
-
- assertThat(cluster.getEndpoints().size(), is(0));
- assertThat(cluster.getRoute(), nullValue());
- }
-
- @Test
- public void testConfig() {
- Cluster cluster = new Cluster.Builder()
- .addEndpoint(Endpoint.create("a"))
- .addEndpoint(Endpoint.create("b"))
- .setRoute("blah")
- .build();
-
- assertThat(cluster.getEndpoints().size(), is(2));
- assertThat(cluster.getEndpoints().get(0).getHostname(), equalTo("a"));
- assertThat(cluster.getEndpoints().get(1).getHostname(), equalTo("b"));
- assertThat(cluster.getRoute(), equalTo("blah"));
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java
deleted file mode 100644
index 3fcc73e3cdc..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.config;
-
-import org.junit.Test;
-
-import javax.net.ssl.SSLContext;
-import java.security.NoSuchAlgorithmException;
-import java.util.Iterator;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.core.Is.is;
-import static org.hamcrest.core.IsNull.nullValue;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-
-/**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- * @since 5.1.22
- */
-public class ConnectionParamsTest {
-
- @Test
- public void testDefaults() {
- ConnectionParams params = new ConnectionParams.Builder().build();
-
- assertThat(params.getHeaders().isEmpty(), is(true));
- assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(1));
- assertThat(params.getSslContext(), nullValue());
- }
-
- @Test
- public void testSetters() throws NoSuchAlgorithmException {
- ConnectionParams params = new ConnectionParams.Builder()
- .addHeader("Foo", "Bar")
- .addHeader("Foo", "Baz")
- .addHeader("Banana", "Apple")
- .setNumPersistentConnectionsPerEndpoint(2)
- .setSslContext(SSLContext.getDefault())
- .build();
-
- assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(2));
-
- assertThat(params.getHeaders().isEmpty(), is(false));
- assertThat(params.getHeaders().size(), is(3));
- //Iteration order seems stable, let's keep it like this for now
- Iterator<Map.Entry<String, String>> headers = params.getHeaders().iterator();
- Map.Entry<String, String> header1 = headers.next();
- assertThat(header1.getKey(), equalTo("Foo"));
- assertThat(header1.getValue(), equalTo("Bar"));
- Map.Entry<String, String> header2 = headers.next();
- assertThat(header2.getKey(), equalTo("Foo"));
- assertThat(header2.getValue(), equalTo("Baz"));
- Map.Entry<String, String> header3 = headers.next();
- assertThat(header3.getKey(), equalTo("Banana"));
- assertThat(header3.getValue(), equalTo("Apple"));
- }
-
- @Test
- public void header_providers_are_registered() {
- ConnectionParams.HeaderProvider dummyProvider1 = () -> "fooValue";
- ConnectionParams.HeaderProvider dummyProvider2 = () -> "barValue";
- ConnectionParams params = new ConnectionParams.Builder()
- .addDynamicHeader("foo", dummyProvider1)
- .addDynamicHeader("bar", dummyProvider2)
- .build();
- Map<String, ConnectionParams.HeaderProvider> headerProviders = params.getDynamicHeaders();
- assertEquals(2, headerProviders.size());
- assertEquals(dummyProvider1, headerProviders.get("foo"));
- assertEquals(dummyProvider2, headerProviders.get("bar"));
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java
deleted file mode 100644
index 415a7746b3a..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.config;
-
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.not;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
-/**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- * @since 5.1.22
- */
-public class EndpointTest {
-
- @Test
- public void testBasic() {
- Endpoint endpoint = Endpoint.create("foo");
-
- assertThat(endpoint.getHostname(), equalTo("foo"));
- assertThat(endpoint.getPort(), equalTo(4080));
- assertThat(endpoint.isUseSsl(), is(false));
- }
-
- @Test
- public void testBasicWithHttpProtocolPrefix() {
- Endpoint endpoint = Endpoint.create("http://foo");
- assertThat(endpoint.getHostname(), equalTo("foo"));
- }
-
- @Test(expected = RuntimeException.class)
- public void testBasicWithHttpsProtocolPrefix() {
- Endpoint.create("https://foo");
- }
-
- @Test
- public void testAdvanced() {
- Endpoint endpoint = Endpoint.create("bar", 1234, true);
-
- assertThat(endpoint.getHostname(), equalTo("bar"));
- assertThat(endpoint.getPort(), equalTo(1234));
- assertThat(endpoint.isUseSsl(), is(true));
- }
-
- @Test
- public void testMethods() {
- Endpoint a = Endpoint.create("a");
- Endpoint b = Endpoint.create("b");
-
- assertThat(a, not(equalTo(b)));
- assertThat(a.hashCode(), not(equalTo(b.hashCode())));
-
- Endpoint a2 = Endpoint.create("a");
-
- assertThat(a, equalTo(a2));
- assertThat(a.hashCode(), equalTo(a2.hashCode()));
-
- a.toString();
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java
deleted file mode 100644
index 82e0b4deac0..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.config;
-
-import org.junit.Test;
-
-import java.util.concurrent.TimeUnit;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
-/**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- * @since 5.1.22
- */
-public class FeedParamsTest {
-
- @Test
- public void testDefaults() {
- FeedParams params = new FeedParams.Builder().build();
-
- assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.JSON_UTF8));
- assertThat(params.getMaxChunkSizeBytes(), is(50 * 1024));
- assertThat(params.getRoute(), nullValue());
- assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(180L));
- assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(20L));
- }
-
- @Test
- public void testConfig() {
- FeedParams params = new FeedParams.Builder()
- .setDataFormat(FeedParams.DataFormat.XML_UTF8)
- .setMaxChunkSizeBytes(123)
- .setRoute("abc")
- .setClientTimeout(321, TimeUnit.SECONDS)
- .build();
-
- assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.XML_UTF8));
- assertThat(params.getMaxChunkSizeBytes(), is(123));
- assertThat(params.getRoute(), equalTo("abc"));
- assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(180L));
- assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(321L));
-
- params = new FeedParams.Builder()
- .setServerTimeout(333L, TimeUnit.SECONDS)
- .setClientTimeout(222L, TimeUnit.SECONDS)
- .build();
-
- assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(333L));
- assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(222L));
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java
deleted file mode 100644
index 8f707f2426e..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core;
-
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
-import java.nio.ReadOnlyBufferException;
-import java.time.Clock;
-
-import static org.junit.Assert.assertEquals;
-
-public class DocumentTest {
-
- @Test
- public void simpleCaseOk() {
- String docId = "doc id";
- String docContent = "foo";
- Document document = new Document(docId, docContent.getBytes(), null, Clock.systemUTC().instant());
- assertEquals(docId, document.getDocumentId());
- assertEquals(ByteBuffer.wrap(docContent.getBytes()), document.getData());
- assertEquals(docContent, document.getDataAsString().toString());
- // Make sure that data is not modified on retrieval.
- assertEquals(docContent, document.getDataAsString().toString());
- assertEquals(ByteBuffer.wrap(docContent.getBytes()), document.getData());
- assertEquals(docId, document.getDocumentId());
- }
-
- @Test(expected = ReadOnlyBufferException.class)
- public void notMutablePutTest() {
- Document document = new Document("id", null, "data", null, Clock.systemUTC().instant());
- document.getData().put("a".getBytes());
- }
-
- @Test(expected = ReadOnlyBufferException.class)
- public void notMutableCompactTest() {
- Document document = new Document("id", null, "data", null, Clock.systemUTC().instant());
- document.getData().compact();
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java
deleted file mode 100644
index aa926578a94..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java
+++ /dev/null
@@ -1,240 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core;
-
-import static org.junit.Assert.*;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * Functional tests for encoding Encoder, i.e. encoding scheme only producing
- * ASCII and never containing white space or control characters.
- *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
- */
-public class EncoderTestCase {
- private final String basic = "abc123";
- private final String basic2 = "abc{20}123";
- private final String quotedIsLast = "abc{20}";
- private final String quotedIsLastDecoded = "abc ";
- private final String basic2Decoded = "abc 123";
- private final String unterminated = "abc{33";
- private final String unterminated2 = "abc{";
- private final String emptyQuoted = "abc{}123";
- private final String outsideUnicode = "abc{7fffffff}";
- private final String noise = "abc{7fff{||\\ffff}";
- private final String fullAsciiEncoded = "{0}{1}{2}{3}{4}{5}{6}{7}{8}{9}"
- + "{a}{b}{c}{d}{e}{f}{10}{11}{12}{13}{14}{15}{16}{17}"
- + "{18}{19}{1a}{1b}{1c}{1d}{1e}{1f}{20}"
- + "!\"#$%&'()*+,-./0123456789:;<=>?@"
- + "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`"
- + "abcdefghijklmnopqrstuvwxyz{7b}|{7d}~{7f}";
- private final int[] testCodepoints = { 0x0, '\n', ' ', 'a', '{', '}', 0x7f, 0x80,
- 0x7ff, 0x800, 0xd7ff, 0xe000, 0xffff, 0x10000, 0x10ffff, 0x34,
- 0x355, 0x2567, 0xfff, 0xe987, 0x100abc };
- private final String semiNastyEncoded = "{0}{a}{20}a{7b}{7d}{7f}{80}"
- + "{7ff}{800}{d7ff}{e000}{ffff}{10000}{10ffff}4"
- + "{355}{2567}{fff}{e987}{100abc}";
- private final String invalidUnicode = "abc\ud812";
- private final String invalidUnicodeEncoded = "abc{d812}";
-
- StringBuilder s;
-
- @Before
- public void setUp() {
- s = new StringBuilder();
- }
-
- @After
- public void tearDown() {
- s = null;
- }
-
- @Test
- public final void testBasic() {
- Encoder.encode(basic, s);
- assertEquals(basic, s.toString());
- }
-
- @Test
- public final void testBasic2() {
- Encoder.encode(basic2Decoded, s);
- assertEquals(basic2, s.toString());
- }
-
- @Test
- public final void testEncodeAscii() {
- Encoder.encode(fullAscii(), s);
- assertEquals(fullAsciiEncoded, s.toString());
- }
-
- @Test
- public final void testEncodeMixed() {
- Encoder.encode(semiNasty(), s);
- assertEquals(semiNastyEncoded, s.toString());
- }
-
- @Test
- public final void testEncodeQuotedIsLast() {
- Encoder.encode(quotedIsLastDecoded, s);
- assertEquals(quotedIsLast, s.toString());
- }
-
- @Test
- public final void testInvalidUnicode() {
- Encoder.encode(invalidUnicode, s);
- assertEquals(invalidUnicodeEncoded, s.toString());
- }
-
-
- @Test
- public final void testDecodeBasic() {
- Encoder.decode(basic, s);
- assertEquals(basic, s.toString());
- }
-
- @Test
- public final void testDecodeBasic2() {
- Encoder.decode(basic2, s);
- assertEquals(basic2Decoded, s.toString());
- }
-
- @Test
- public final void testDecodeAscii() {
- Encoder.decode(fullAsciiEncoded, s);
- assertEquals(fullAscii(), s.toString());
- }
-
- @Test
- public final void testDecodeMixed() {
- Encoder.decode(semiNastyEncoded, s);
- assertEquals(semiNasty(), s.toString());
- }
-
-
-
- @Test
- public final void testDecodeQuotedIsLast() {
- Encoder.decode(quotedIsLast, s);
- assertEquals(quotedIsLastDecoded, s.toString());
- }
-
-
- @Test
- public final void testDecodeUnterminated() {
- try {
- Encoder.decode(unterminated, s);
- } catch (IllegalArgumentException e) {
- return;
- }
- fail("Expected IllegalArgumentException");
- }
-
- @Test
- public final void testDecodeUnterminated2() {
- try {
- Encoder.decode(unterminated2, s);
- } catch (IllegalArgumentException e) {
- return;
- }
- fail("Expected IllegalArgumentException");
-
- }
-
- @Test
- public final void testEmptyQuoted() {
- try {
- Encoder.decode(emptyQuoted, s);
- } catch (IllegalArgumentException e) {
- return;
- }
- fail("Expected IllegalArgumentException");
- }
-
- @Test
- public final void testOutsideUnicode() {
- try {
- Encoder.decode(outsideUnicode, s);
- } catch (IllegalArgumentException e) {
- return;
- }
- fail("Expected IllegalArgumentException");
- }
-
-
- @Test
- public final void testNoise() {
- try {
- Encoder.decode(noise, s);
- } catch (IllegalArgumentException e) {
- return;
- }
- fail("Expected IllegalArgumentException");
- }
-
- @Test
- public final void testIllegalInputCharacter() {
- try {
- Encoder.decode("abc\u00e5", s);
- } catch (IllegalArgumentException e) {
- return;
- }
- fail("Expected IllegalArgumentException");
- }
-
-
- @Test
- public final void testNoIllegalCharactersInOutputForAscii() {
- Encoder.encode(fullAscii(), s);
- checkNoNonAscii(s.toString());
- }
-
- @Test
- public final void testNoIllegalCharactersInOutputForMixedInput() {
- Encoder.encode(semiNasty(), s);
- checkNoNonAscii(s.toString());
- }
-
- @Test
- public final void testSymmetryAscii() {
- StringBuilder forDecoding = new StringBuilder();
- Encoder.encode(fullAscii(), s);
- Encoder.decode(s.toString(), forDecoding);
- assertEquals(fullAscii(), forDecoding.toString());
- }
-
- @Test
- public final void testSymmetryMixed() {
- StringBuilder forDecoding = new StringBuilder();
- Encoder.encode(semiNasty(), s);
- Encoder.decode(s.toString(), forDecoding);
- assertEquals(semiNasty(), forDecoding.toString());
- }
-
-
- private void checkNoNonAscii(String input) {
- for (int i = 0; i < input.length(); ++i) {
- char c = input.charAt(i);
- if (c > '~' || c <= ' ') {
- fail("Encoded data contained character ordinal " + Integer.toHexString(c));
- }
- }
- }
-
- private String fullAscii() {
- StringBuilder s = new StringBuilder();
- for (int i = 0; i <= 0x7f; ++i) {
- s.append((char) i);
- }
- return s.toString();
- }
-
- private String semiNasty() {
- StringBuilder s = new StringBuilder();
- for (int i = 0; i < testCodepoints.length; ++i) {
- s.append(Character.toChars(testCodepoints[i]));
- }
- return s.toString();
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java
deleted file mode 100644
index 2e6efffdd2c..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java
+++ /dev/null
@@ -1,125 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core;
-
-import com.yahoo.vespa.http.client.FeedClient;
-import com.yahoo.vespa.http.client.FeedEndpointException;
-import com.yahoo.vespa.http.client.ManualClock;
-import com.yahoo.vespa.http.client.Result;
-import com.yahoo.vespa.http.client.config.Cluster;
-import com.yahoo.vespa.http.client.config.ConnectionParams;
-import com.yahoo.vespa.http.client.config.Endpoint;
-import com.yahoo.vespa.http.client.config.SessionParams;
-import com.yahoo.vespa.http.client.core.communication.ClusterConnection;
-import com.yahoo.vespa.http.client.core.communication.IOThread;
-import com.yahoo.vespa.http.client.core.communication.IOThreadTest;
-import com.yahoo.vespa.http.client.core.operationProcessor.IncompleteResultsThrottler;
-import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor;
-
-import java.time.Instant;
-import java.util.List;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Helper for testing with an operation processor
- *
- * @author bratseth
- */
-public class OperationProcessorTester {
-
- private final Endpoint endpoint;
- private final int clusterId = 0;
- private final ManualClock clock;
- private final TestResultCallback resultCallback;
- private final OperationProcessor operationProcessor;
-
- public OperationProcessorTester() {
- endpoint = Endpoint.create("test-endpoint");
- SessionParams.Builder params = new SessionParams.Builder();
- Cluster.Builder clusterParams = new Cluster.Builder();
- clusterParams.addEndpoint(endpoint);
- params.addCluster(clusterParams.build());
- ConnectionParams.Builder connectionParams = new ConnectionParams.Builder();
- connectionParams.setDryRun(true);
- connectionParams.setRunThreads(false);
- params.setConnectionParams(connectionParams.build());
-
- clock = new ManualClock(Instant.ofEpochMilli(0));
- resultCallback = new TestResultCallback();
- operationProcessor = new OperationProcessor(new IncompleteResultsThrottler(1, 100, clock, new ThrottlePolicy()),
- resultCallback,
- params.build(),
- new ScheduledThreadPoolExecutor(1),
- clock);
- }
-
- public ManualClock clock() { return clock; }
-
- /** Asserts that this has but a single IOThread and returns it */
- public IOThread getSingleIOThread() {
- assertEquals(1, clusterConnections().size());
- assertEquals(1, clusterConnections().get(0).ioThreads().size());
- return clusterConnections().get(0).ioThreads().get(0);
- }
-
- /** Do n iteration of work in all io threads of this */
- public void tick(int n) {
- for (int i = 0; i < n; i++)
- for (ClusterConnection cluster : operationProcessor.clusters())
- for (IOThread thread : cluster.ioThreads())
- thread.tick();
- }
-
- public void send(String documentId) {
- operationProcessor.sendDocument(new Document(documentId, documentId, "data of " + documentId, null, clock.instant()));
- }
-
- public int incomplete() {
- return operationProcessor.getIncompleteResultQueueSize();
- }
-
- public int success() {
- return resultCallback.successes;
- }
-
- public List<ClusterConnection> clusterConnections() {
- return operationProcessor.clusters();
- }
-
- public int failures() {
- return resultCallback.failures;
- }
-
- public int endpointExceptions() {
- return resultCallback.endpointExceptions;
- }
-
- public Result lastResult() {
- return resultCallback.lastResult;
- }
-
- private static class TestResultCallback implements FeedClient.ResultCallback {
-
- private int successes = 0;
- private int failures = 0;
- private int endpointExceptions = 0;
- private Result lastResult;
-
- @Override
- public void onCompletion(String docId, Result documentResult) {
- this.lastResult = documentResult;
- if (documentResult.isSuccess())
- successes++;
- else
- failures++;
- }
-
- @Override
- public void onEndpointException(FeedEndpointException exception) {
- endpointExceptions++;
- }
-
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java
deleted file mode 100644
index d5a4d09fb8f..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core;
-
-import org.junit.Test;
-
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
-
-public class ThrottlePolicyTest {
-
- private final ThrottlePolicy throttlePolicy = new ThrottlePolicy();
- // Default values for tests.
- private int numOk = 1000;
- private int prevOk = 1000;
- private int prevMax = 100;
- private int max = 100;
- private boolean queued = true;
- private double dynamicFactor = 0.1;
-
- @Test
- public void samePerformanceShouldTuneDown() {
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(95));
- }
-
- @Test
- public void improvedPerformanceSameSizeShouldTuneDown() {
- numOk += 200;
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(89));
- }
-
- @Test
- public void improvedPerformanceSmallerSizeTuneDownFurther() {
- numOk += 200;
- max = 70;
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(63));
- }
-
- @Test
- public void improvedPerformanceLargerSizeIncrease() {
- numOk += 200;
- max = 130;
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(143));
- dynamicFactor = 100;
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(156));
- }
-
- @Test
- public void improvedPerformanceLargerSizeButQueuedFalse() {
- numOk += 200;
- max = 130;
- queued = false;
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(128));
- }
-
- @Test
- public void lowerPerformanceSameSizeShouldIncrease() {
- numOk -= 200;
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(110));
- }
-
- @Test
- public void lowerPerformanceSmallerSizeShouldIncreaseSize() {
- numOk -= 200;
- max = 30;
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(33));
- dynamicFactor = 100;
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(36));
- }
-
- @Test
- public void lowerPerformanceLargerSizeTuneDownFurther() {
- numOk -= 200;
- max = 130;
- assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(116));
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java
deleted file mode 100644
index e71cee0f4c2..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java
+++ /dev/null
@@ -1,254 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.dataformat.xml.XmlMapper;
-import com.yahoo.vespa.http.client.FeedClient;
-import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.xml.sax.SAXParseException;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.Is.is;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-
-public class XmlFeedReaderTest {
- private final static String feedResource = "/vespacorpfeed-prod-sample.xml";
-
- private final static String feedResource2 = "/xml-challenge.xml";
- private final static String feedResource3 = "/xml-challenge2.xml";
- private final static String feedResource4 = "/xml-challenge3.xml";
-
- private static final XmlMapper xmlMapper = new XmlMapper();
-
- private final String updateDocUpdate =
- "<?xml version=\"1.0\"?>\n" +
- "<vespafeed>\n" +
- "<update documentid=\"id:banana:banana::complex\" documenttype=\"banana\">\n" +
- " <add fieldpath=\"structarr\">\n" +
- " <item>\n" +
- " <bytearr>\n" +
- " <item>30</item>\n" +
- " <item>55</item>\n" +
- " </bytearr>\n" +
- " </item>\n" +
- " </add>\n" +
- "</update>\n" +
- "</vespafeed>\n";
-
- @Test
- public void testReadUpdate() throws Exception {
- InputStream stream = new ByteArrayInputStream(updateDocUpdate.getBytes(StandardCharsets.UTF_8));
- AtomicInteger numSent = new AtomicInteger(0);
- FeedClient feedClient = mock(FeedClient.class);
- XmlFeedReader.read(stream, feedClient, numSent);
- assertThat(numSent.get(), is(1));
- }
-
- private final String updateDocRemove =
- "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
- "\n" +
- "<vespafeed>\n" +
- " <remove documentid=\"id:music:music::http://music.yahoo.com/Bob0/BestOf\" />\n" +
- " <remove documentid=\"id:music:music::http://music.yahoo.com/Bob9/BestOf\" />\n" +
- "</vespafeed>";
-
- @Test
- public void testReadRemove() throws Exception {
- InputStream stream = new ByteArrayInputStream(updateDocRemove.getBytes(StandardCharsets.UTF_8));
- AtomicInteger numSent = new AtomicInteger(0);
- FeedClient feedClient = mock(FeedClient.class);
- XmlFeedReader.read(stream, feedClient, numSent);
- assertThat(numSent.get(), is(2));
- }
-
- private final String insertDocOperation = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"+
- "<vespafeed>\n"+
- "\n"+
- " <document type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/bobdylan/BestOf\">\n"+
- " <title>Best of Bob Dylan</title>\n"+
- " </document>\n"+
- "\n"+
- " <document type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/metallica/BestOf\">\n"+
- " <title>Best of Metallica</title>\n"+
- " </document>\n"+
- "</vespafeed>";
-
- @Test
- public void testInsert() throws Exception {
- InputStream stream = new ByteArrayInputStream(insertDocOperation.getBytes(StandardCharsets.UTF_8));
- AtomicInteger numSent = new AtomicInteger(0);
- FeedClient feedClient = mock(FeedClient.class);
- XmlFeedReader.read(stream, feedClient, numSent);
- assertThat(numSent.get(), is(2));
- }
-
- private final String badperation = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"+
- "<vespafeed>\n"+
- " <badtag type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/bobdylan/BestOf\">\n"+
- " <title>Best of Bob Dylan</title>\n"+
- " </badtag>\n"+
- "</vespafeed>";
-
- @Test
- public void testNonDocument() throws Exception {
- InputStream stream = new ByteArrayInputStream(badperation.getBytes(StandardCharsets.UTF_8));
- AtomicInteger numSent = new AtomicInteger(0);
- FeedClient feedClient = mock(FeedClient.class);
- XmlFeedReader.read(stream, feedClient, numSent);
- assertThat(numSent.get(), is(0));
- }
-
- @Test(expected=SAXParseException.class)
- public void testGarbage() throws Exception {
- InputStream stream = new ByteArrayInputStream("eehh".getBytes(StandardCharsets.UTF_8));
- AtomicInteger numSent = new AtomicInteger(0);
- FeedClient feedClient = mock(FeedClient.class);
- XmlFeedReader.read(stream, feedClient, numSent);
- }
-
- @Test
- public void testEncoding() throws Exception {
- InputStream stream = new ByteArrayInputStream("<?xml version=\"1.0\" encoding=\"utf8\"?><vespafeed><remove documentid=\"id:&amp;\"/></vespafeed>"
- .getBytes(StandardCharsets.UTF_8));
- AtomicInteger numSent = new AtomicInteger(0);
- FeedClient feedClient = mock(FeedClient.class);
-
- doAnswer(new Answer<Object>() {
- public Object answer(InvocationOnMock invocation) {
- Object[] args = invocation.getArguments();
- String docId = (String) args[0];
- CharSequence value = (CharSequence)args[1];
- assertThat(value.toString(), is("<remove documentid=\"id:&amp;\"></remove>"));
- assertThat(docId, is("id:&"));
- return null;
- }
- }).when(feedClient).stream(anyString(), any());
- XmlFeedReader.read(stream, feedClient, numSent);
- assertThat(numSent.get(), is(1));
- }
-
- private final String characterDocs = "" +
- "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
- "<!-- GENERATED VESPA-XML BY YSSTOXML -->\n" +
- "<!-- USE ONLY FOR BATCH INDEXING -->\n" +
- "<vespafeed>\n" +
- " <document documenttype=\"simple\" documentid=\"id:test::&amp;http://www.e.no/matprat\">\n" +
- " <language><![CDATA[ja]]></language>\n" +
- " <title><![CDATA[test document1]]></title>\n" +
- " <description><![CDATA[Bjørnen' blåbær på øy nærheten.]]></description>\n" +
- " <date>1091356845</date>\n" +
- " <surl><![CDATA[http://www.eventyr.no/matprat]]></surl>\n" +
- " </document>\n" +
- "\n" +
- "</vespafeed>\n";
-
- @Test
- public void testCharacterEndcoding() throws Exception {
- InputStream stream = new ByteArrayInputStream(characterDocs.getBytes(StandardCharsets.UTF_8));
- AtomicInteger numSent = new AtomicInteger(0);
- FeedClient feedClient = mock(FeedClient.class);
- final AtomicBoolean success = new AtomicBoolean(false);
- doAnswer(new Answer<Object>() {
- public Object answer(InvocationOnMock invocation) {
- Object[] args = invocation.getArguments();
- String docId = (String) args[0];
- CharSequence value = (CharSequence)args[1];
- assertThat(value.toString(), is(
- "<document documenttype=\"simple\" documentid=\"id:test::&amp;http://www.e.no/matprat\">\n" +
- " <language><![CDATA[ja]]></language>\n" +
- " <title><![CDATA[test document1]]></title>\n" +
- " <description><![CDATA[Bjørnen' blåbær på øy nærheten.]]></description>\n" +
- " <date>1091356845</date>\n" +
- " <surl><![CDATA[http://www.eventyr.no/matprat]]></surl>\n" +
- " </document>"));
- success.set(true);
- return null;
- }
- }).when(feedClient).stream(anyString(), any());
- XmlFeedReader.read(stream, feedClient, numSent);
- assertThat(numSent.get(), is(1));
- assert(success.get());
- }
-
- @Test
- public void testRealData() throws Exception {
- InputStream inputStream = XmlFeedReaderTest.class.getResourceAsStream(feedResource);
- BufferedInputStream bis = new BufferedInputStream(inputStream);
- AtomicInteger numSent = new AtomicInteger(0);
- FeedClient feedClient = mock(FeedClient.class);
-
- XmlFeedReader.read(bis, feedClient, numSent);
- assertThat(numSent.get(), is(6));
- }
-
- private static class XmlTestFeedClient implements FeedClient {
-
- public List<String> documentIds = new ArrayList<>();
- public List<CharSequence> datas = new ArrayList<>();
- public List<Object> contexts = new ArrayList<>();
-
- @Override
- public void stream(String documentId, CharSequence documentData) {
- stream(documentId, documentData, null);
- }
-
- @Override
- public void stream(String documentId, String operationId, CharSequence documentData, Object context) {
- documentIds.add(documentId);
- datas.add(documentData);
- contexts.add(context);
- }
-
-
- @Override
- public void close() { }
-
- @Override
- public String getStatsAsJson() { return null; }
- }
-
- // Only for xml with single doc.
- private void verifyNoTransformationOfXml(String filename) throws Exception {
- InputStream inputStream = XmlFeedReaderTest.class.getResourceAsStream(filename);
- BufferedInputStream bis = new BufferedInputStream(inputStream);
- AtomicInteger numSent = new AtomicInteger(0);
- XmlTestFeedClient feedClient = new XmlTestFeedClient();
- XmlFeedReader.read(bis, feedClient, numSent);
- assertThat(numSent.get(), is(1));
- String document = feedClient.datas.get(0).toString();
-
- InputStream inputStream2 = XmlFeedReaderTest.class.getResourceAsStream(filename);
- String rawXML = new java.util.Scanner(inputStream2, "UTF-8").useDelimiter("\\A").next();
-
- JsonNode decodedDocument = xmlMapper.readTree(document);
- JsonNode rawDocuments = xmlMapper.readTree(rawXML);
- assertThat(decodedDocument, is(rawDocuments.get("document")));
- }
-
- @Test public void testCData() throws Exception {
- verifyNoTransformationOfXml(feedResource2);
- }
-
- @Test public void testPCData() throws Exception {
- verifyNoTransformationOfXml(feedResource3);
- }
-
- @Test public void testAposData() throws Exception {
- verifyNoTransformationOfXml(feedResource4);
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java
deleted file mode 100644
index 283c1169440..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.api;
-
-import org.junit.Test;
-
-import java.time.Instant;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.*;
-
-/**
- * @author dybis
- */
-public class FeedClientImplTest {
-
- int sleepValueMillis = 1;
-
- @Test
- public void testCloseWaitTimeOldTimestamp() {
- assertThat(FeedClientImpl.waitForOperations(Instant.now().minusSeconds(1000), sleepValueMillis, 10), is(false));
- }
-
- @Test
- public void testCloseWaitTimeOutInFutureStillOperations() {
- assertThat(FeedClientImpl.waitForOperations(Instant.now(), sleepValueMillis, 2000), is(true));
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java
deleted file mode 100644
index 2597ddddd88..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java
+++ /dev/null
@@ -1,355 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.communication;
-
-import com.yahoo.vespa.http.client.TestUtils;
-import com.yahoo.vespa.http.client.config.ConnectionParams;
-import com.yahoo.vespa.http.client.config.Endpoint;
-import com.yahoo.vespa.http.client.config.FeedParams;
-import com.yahoo.vespa.http.client.core.Document;
-import com.yahoo.vespa.http.client.core.Headers;
-import com.yahoo.vespa.http.client.core.ServerResponseException;
-import org.apache.http.Header;
-import org.apache.http.HeaderElement;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.ParseException;
-import org.apache.http.StatusLine;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.message.BasicHeader;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.mockito.stubbing.Answer;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.time.Clock;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class ApacheGatewayConnectionTest {
-
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- @Test
- public void testProtocolV3() throws Exception {
- Endpoint endpoint = Endpoint.create("localhost", 666, false);
- FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build();
- String clusterSpecificRoute = "";
- ConnectionParams connectionParams = new ConnectionParams.Builder().build();
- List<Document> documents = new ArrayList<>();
-
- String vespaDocContent = "Hello, I a JSON doc.";
- String docId = "42";
-
- // This is the fake server, takes header client ID and uses this as session Id.
- ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> {
- Header clientIdHeader = post.getFirstHeader(Headers.CLIENT_ID);
- return httpResponse(clientIdHeader.getValue(), "3");
- });
-
- ApacheGatewayConnection apacheGatewayConnection =
- new ApacheGatewayConnection(
- endpoint,
- feedParams,
- clusterSpecificRoute,
- connectionParams,
- mockFactory,
- "clientId",
- Clock.systemUTC());
- apacheGatewayConnection.connect();
- apacheGatewayConnection.handshake();
- documents.add(createDoc(docId, vespaDocContent, true));
-
- apacheGatewayConnection.write(documents);
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testServerReturnsBadSessionInV3() throws Exception {
- Endpoint endpoint = Endpoint.create("localhost", 666, false);
- FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build();
- String clusterSpecificRoute = "";
- ConnectionParams connectionParams = new ConnectionParams.Builder().build();
-
- // This is the fake server, returns wrong session Id.
- ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> httpResponse("Wrong Id from server", "3"));
-
- ApacheGatewayConnection apacheGatewayConnection =
- new ApacheGatewayConnection(
- endpoint,
- feedParams,
- clusterSpecificRoute,
- connectionParams,
- mockFactory,
- "clientId",
- Clock.systemUTC());
- apacheGatewayConnection.connect();
- List<Document> documents = new ArrayList<>();
- apacheGatewayConnection.write(documents);
- }
-
- @Test
- public void testJsonDocumentHeader() throws Exception {
- Endpoint endpoint = Endpoint.create("localhost", 666, false);
- FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build();
- String clusterSpecificRoute = "";
- ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build();
- List<Document> documents = new ArrayList<>();
-
- String vespaDocContent ="Hello, I a JSON doc.";
- String docId = "42";
-
- AtomicInteger requestsReceived = new AtomicInteger(0);
-
- // This is the fake server, checks that DATA_FORMAT header is set properly.
- ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> {
- Header header = post.getFirstHeader(Headers.DATA_FORMAT);
- if (requestsReceived.incrementAndGet() == 1) {
- // This is handshake, it is not json.
- assert (header == null);
- return httpResponse("clientId", "3");
- }
- assertNotNull(header);
- assertEquals(FeedParams.DataFormat.JSON_UTF8.name(), header.getValue());
- // Test is done.
- return httpResponse("clientId", "3");
- });
-
- ApacheGatewayConnection apacheGatewayConnection =
- new ApacheGatewayConnection(
- endpoint,
- feedParams,
- clusterSpecificRoute,
- connectionParams,
- mockFactory,
- "clientId",
- Clock.systemUTC());
- apacheGatewayConnection.connect();
- apacheGatewayConnection.handshake();
-
- documents.add(createDoc(docId, vespaDocContent, true));
-
- apacheGatewayConnection.write(documents);
- }
-
- @Test
- public void testZipAndCreateEntity() throws IOException {
- String testString = "Hello world";
- InputStream stream = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8));
- // Send in test data to method.
- InputStreamEntity inputStreamEntity = ApacheGatewayConnection.zipAndCreateEntity(stream);
- // Verify zipped data by comparing unzipped data with test data.
- String rawContent = TestUtils.zipStreamToString(inputStreamEntity.getContent());
- assertEquals(testString, rawContent);
- }
-
- /**
- * Mocks the HttpClient, and verifies that the compressed data is sent.
- */
- @Test
- public void testCompressedWriteOperations() throws Exception {
- Endpoint endpoint = Endpoint.create("localhost", 666, false);
- FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.XML_UTF8).build();
- String clusterSpecificRoute = "";
- ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build();
- List<Document> documents = new ArrayList<>();
-
- String vespaDocContent ="Hello, I am the document data.";
- String docId = "42";
-
- Document doc = createDoc(docId, vespaDocContent, false);
-
- // When sending data on http client, check if it is compressed. If compressed, unzip, check result,
- // and count down latch.
- ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> {
- Header header = post.getFirstHeader("Content-Encoding");
- if (header != null && header.getValue().equals("gzip")) {
- final String rawContent = TestUtils.zipStreamToString(post.getEntity().getContent());
- final String vespaHeaderText = "<vespafeed>\n";
- final String vespaFooterText = "</vespafeed>\n";
-
- assertEquals(doc.getOperationId() + " 38\n" + vespaHeaderText + vespaDocContent + "\n" + vespaFooterText,
- rawContent);
- }
- return httpResponse("clientId", "3");
- });
-
- StatusLine statusLineMock = mock(StatusLine.class);
- when(statusLineMock.getStatusCode()).thenReturn(200);
-
- ApacheGatewayConnection apacheGatewayConnection =
- new ApacheGatewayConnection(
- endpoint,
- feedParams,
- clusterSpecificRoute,
- connectionParams,
- mockFactory,
- "clientId",
- Clock.systemUTC());
- apacheGatewayConnection.connect();
- apacheGatewayConnection.handshake();
-
- documents.add(doc);
-
- apacheGatewayConnection.write(documents);
- }
-
- @Test
- public void dynamic_headers_are_added_to_the_response() throws IOException, ServerResponseException {
- ConnectionParams.HeaderProvider headerProvider = mock(ConnectionParams.HeaderProvider.class);
- when(headerProvider.getHeaderValue())
- .thenReturn("v1")
- .thenReturn("v2")
- .thenReturn("v3");
-
- ConnectionParams connectionParams = new ConnectionParams.Builder()
- .addDynamicHeader("foo", headerProvider)
- .build();
-
- AtomicInteger counter = new AtomicInteger(1);
- ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> {
- Header[] fooHeader = post.getHeaders("foo");
- assertEquals(1, fooHeader.length);
- assertEquals("foo", fooHeader[0].getName());
- assertEquals("v" + counter.getAndIncrement(), fooHeader[0].getValue());
- return httpResponse("clientId", "3");
-
- });
-
- ApacheGatewayConnection apacheGatewayConnection =
- new ApacheGatewayConnection(
- Endpoint.create("localhost", 666, false),
- new FeedParams.Builder().build(),
- "",
- connectionParams,
- mockFactory,
- "clientId",
- Clock.systemUTC());
- apacheGatewayConnection.connect();
- apacheGatewayConnection.handshake();
-
- List<Document> documents = new ArrayList<>();
- documents.add(createDoc("42", "content", true));
- apacheGatewayConnection.write(documents);
- apacheGatewayConnection.write(documents);
-
- verify(headerProvider, times(3)).getHeaderValue(); // 1x connect(), 2x writeOperations()
- }
-
- @Test
- public void detailed_error_message_is_extracted_from_error_responses_with_json() throws IOException, ServerResponseException {
- String reasonPhrase = "Unauthorized";
- String errorMessage = "Invalid credentials";
- expectedException.expect(ServerResponseException.class);
- expectedException.expectMessage(reasonPhrase + " - " + errorMessage);
-
- ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> createErrorHttpResponse(401, reasonPhrase, errorMessage));
-
- ApacheGatewayConnection apacheGatewayConnection =
- new ApacheGatewayConnection(
- Endpoint.create("localhost", 666, false),
- new FeedParams.Builder().build(),
- "",
- new ConnectionParams.Builder().build(),
- mockFactory,
- "clientId",
- Clock.systemUTC());
- apacheGatewayConnection.connect();
- apacheGatewayConnection.handshake();
-
- apacheGatewayConnection.write(Collections.singletonList(createDoc("42", "content", true)));
- }
-
- private static ApacheGatewayConnection.HttpClientFactory mockHttpClientFactory(HttpExecuteMock httpExecuteMock) throws IOException {
- ApacheGatewayConnection.HttpClientFactory mockFactory =
- mock(ApacheGatewayConnection.HttpClientFactory.class);
- CloseableHttpClient httpClientMock = mock(CloseableHttpClient.class);
- when(mockFactory.createClient()).thenReturn(httpClientMock);
- when(httpClientMock.execute(any())).thenAnswer((Answer) invocation -> {
- Object[] args = invocation.getArguments();
- HttpPost post = (HttpPost) args[0];
- return httpExecuteMock.execute(post);
- });
- return mockFactory;
- }
-
- @FunctionalInterface private interface HttpExecuteMock {
- HttpResponse execute(HttpPost httpPost) throws IOException;
- }
-
- private Document createDoc(String docId, String content, boolean useJson) {
- return new Document(docId, content.getBytes(), null, Clock.systemUTC().instant());
- }
-
- private void addMockedHeader(HttpResponse httpResponseMock, String name, String value, HeaderElement[] elements) {
- Header header = new Header() {
- @Override
- public String getName() {
- return name;
- }
- @Override
- public String getValue() {
- return value;
- }
- @Override
- public HeaderElement[] getElements() throws ParseException {
- return elements;
- }
- };
- when(httpResponseMock.getFirstHeader(name)).thenReturn(header);
- }
-
- private HttpResponse httpResponse(String sessionIdInResult, String version) throws IOException {
- CloseableHttpResponse httpResponseMock = mock(CloseableHttpResponse.class);
-
- StatusLine statusLineMock = mock(StatusLine.class);
- when(httpResponseMock.getStatusLine()).thenReturn(statusLineMock);
- when(statusLineMock.getStatusCode()).thenReturn(200);
-
- addMockedHeader(httpResponseMock, Headers.SESSION_ID, sessionIdInResult, null);
- addMockedHeader(httpResponseMock, Headers.VERSION, version, null);
- HeaderElement[] headerElements = new HeaderElement[1];
- headerElements[0] = mock(HeaderElement.class);
-
- final HttpEntity httpEntityMock = mock(HttpEntity.class);
- when(httpResponseMock.getEntity()).thenReturn(httpEntityMock);
-
- final InputStream inputs = new ByteArrayInputStream("fake response data".getBytes());
-
- when(httpEntityMock.getContent()).thenReturn(inputs);
- return httpResponseMock;
- }
-
- private static HttpResponse createErrorHttpResponse(int statusCode, String reasonPhrase, String message) throws IOException {
- CloseableHttpResponse response = mock(CloseableHttpResponse.class);
-
- StatusLine statusLine = mock(StatusLine.class);
- when(statusLine.getStatusCode()).thenReturn(statusCode);
- when(statusLine.getReasonPhrase()).thenReturn(reasonPhrase);
- when(response.getStatusLine()).thenReturn(statusLine);
-
- HttpEntity httpEntity = mock(HttpEntity.class);
- when(httpEntity.getContentType()).thenReturn(new BasicHeader("Content-Type", "application/json"));
- String json = String.format("{\"message\": \"%s\"}", message);
- when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream(json.getBytes()));
- when(response.getEntity()).thenReturn(httpEntity);
- return response;
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java
deleted file mode 100644
index 9845768280e..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.communication;
-
-import com.yahoo.vespa.http.client.core.communication.ByteBufferInputStream;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-/**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- * @since 5.1.20
- */
-public class ByteBufferInputStreamTest {
-
- private static ByteBuffer[] getAbcde() {
- ByteBuffer[] buffers = new ByteBuffer[5];
- buffers[0] = ByteBuffer.wrap("a".getBytes(StandardCharsets.UTF_8));
- buffers[1] = ByteBuffer.wrap("b".getBytes(StandardCharsets.UTF_8));
- buffers[2] = ByteBuffer.wrap("c".getBytes(StandardCharsets.UTF_8));
- buffers[3] = ByteBuffer.wrap("d".getBytes(StandardCharsets.UTF_8));
- buffers[4] = ByteBuffer.wrap("e".getBytes(StandardCharsets.UTF_8));
- return buffers;
- }
-
- @Test
- public void requireThatExhaustedBufferWorks() throws IOException {
- ByteBuffer[] buffers = getAbcde();
- buffers[2].get();
- ByteBufferInputStream in = new ByteBufferInputStream(buffers);
-
- byte[] out = new byte[100];
- int pos = 0;
-
- final int GUARD = 1000;
- int i;
- for (i = 0; i < GUARD; i++) {
- int r = in.read();
- if (r == -1) {
- break;
- }
- out[pos] = (byte) (0xFF & r);
- ++pos;
- }
- assertTrue(i < GUARD);
- assertThat(pos, is(4));
-
- String outString = new String(out, 0, pos, StandardCharsets.UTF_8);
- assertThat(outString, equalTo("abde"));
-
- }
-
- @Test
- public void requireThatBulkReadWorks() throws IOException {
- ByteBuffer[] buffers = getAbcde();
- ByteBufferInputStream in = new ByteBufferInputStream(buffers);
-
- byte[] out = new byte[100];
- int pos = 0;
-
- final int GUARD = 1000;
- int i;
- for (i = 0; i < GUARD; i++) {
- int numReadNow;
- if (i == 0) {
- numReadNow = in.read(out);
- } else {
- numReadNow = in.read(out, pos, (out.length - pos));
- }
- if (numReadNow == -1) {
- break;
- }
- pos += numReadNow;
- }
- assertTrue(i < GUARD);
- assertThat(pos, is(5));
-
- String outString = new String(out, 0, pos, StandardCharsets.UTF_8);
- assertThat(outString, equalTo("abcde"));
- }
-
- @Test
- public void requireThatSingleByteReadWorks() throws IOException {
- ByteBuffer[] buffers = getAbcde();
- ByteBufferInputStream in = new ByteBufferInputStream(buffers);
-
- byte[] out = new byte[100];
- int pos = 0;
-
- final int GUARD = 1000;
- int i;
- for (i = 0; i < GUARD; i++) {
- int r = in.read();
- if (r == -1) {
- break;
- }
- out[pos] = (byte) (0xFF & r);
- ++pos;
- }
- assertTrue(i < GUARD);
- assertThat(pos, is(5));
-
- String outString = new String(out, 0, pos, StandardCharsets.UTF_8);
- assertThat(outString, equalTo("abcde"));
- }
-
- @Test
- public void requireThatMarkIsNotSupported() throws IOException {
- ByteBuffer[] buffers = getAbcde();
- ByteBufferInputStream in = new ByteBufferInputStream(buffers);
- assertThat(in.markSupported(), is(false));
- in.mark(0); //a no-op
- }
-
- @Test(expected = IOException.class)
- public void requireThatResetFails() throws IOException {
- ByteBuffer[] buffers = getAbcde();
- ByteBufferInputStream in = new ByteBufferInputStream(buffers);
- in.reset();
- }
-
- @Test(expected = IOException.class)
- public void requireThatSkipFails() throws IOException {
- ByteBuffer[] buffers = getAbcde();
- ByteBufferInputStream in = new ByteBufferInputStream(buffers);
- in.skip(1L);
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java
deleted file mode 100644
index 227f4a5239c..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.communication;
-
-import com.yahoo.vespa.http.client.core.Document;
-import org.junit.Test;
-
-import java.time.Clock;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-public class CloseableQTestCase {
-
- @Test
- public void requestThatPutIsInterruptedOnClose() throws InterruptedException {
- Clock clock = Clock.systemUTC();
- DocumentQueue q = new DocumentQueue(1, clock);
- q.put(new Document("id", null, "data", null, clock.instant()), false);
- Thread t = new Thread(new Runnable() {
- @Override
- public void run() {
- try {
- Thread.sleep(3000);
- } catch (InterruptedException e) {
- }
- q.close();
- q.clear();
- }
- });
- t.start();
- try {
- q.put(new Document("id2", null, "data2", null, Clock.systemUTC().instant()), false);
- fail("This shouldn't have worked.");
- } catch (IllegalStateException ise) {
- // ok!
- }
- try {
- t.join();
- } catch (InterruptedException e) {
- }
- }
-
- @Test
- public void requireThatSelfIsUnbounded() throws InterruptedException {
- DocumentQueue q = new DocumentQueue(1, Clock.systemUTC());
- q.put(new Document("1", null, "data", null, Clock.systemUTC().instant()), true);
- q.put(new Document("2", null, "data", null, Clock.systemUTC().instant()), true);
- q.put(new Document("3", null, "data", null, Clock.systemUTC().instant()), true);
- assertEquals(3, q.size());
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java
deleted file mode 100644
index 78ccfed3dbc..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java
+++ /dev/null
@@ -1,89 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.communication;
-
-import com.yahoo.vespa.http.client.Result;
-import com.yahoo.vespa.http.client.config.Endpoint;
-import com.yahoo.vespa.http.client.core.EndpointResult;
-import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor;
-import org.junit.Test;
-
-import java.time.Clock;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.mock;
-
-/**
- * @author Einar M R Rosenvinge
- */
-public class EndpointResultQueueTest {
-
- @Test
- public void testBasics() {
- Endpoint endpoint = Endpoint.create("a");
-
- GatewayConnection connection = new DryRunGatewayConnection(endpoint, Clock.systemUTC());
- OperationProcessor mockAggregator = mock(OperationProcessor.class);
- final AtomicInteger resultCount = new AtomicInteger(0);
-
- doAnswer(invocationOnMock -> {
- resultCount.getAndIncrement();
- return null;
- }).when(mockAggregator).resultReceived(any(), eq(0));
-
- EndpointResultQueue q = new EndpointResultQueue(
- mockAggregator, endpoint, 0, new ScheduledThreadPoolExecutor(1), 100L * 1000L);
-
- q.operationSent("op1", connection);
- assertThat(q.getPendingSize(), is(1));
- q.operationSent("op2", connection);
- assertThat(q.getPendingSize(), is(2));
- q.operationSent("op3", connection);
- assertThat(q.getPendingSize(), is(3));
- q.resultReceived(new EndpointResult("op1", new Result.Detail(endpoint)), 0);
- assertThat(q.getPendingSize(), is(2));
- q.resultReceived(new EndpointResult("op2", new Result.Detail(endpoint)), 0);
- assertThat(q.getPendingSize(), is(1));
- q.resultReceived(new EndpointResult("op3", new Result.Detail(endpoint)), 0);
- assertThat(q.getPendingSize(), is(0));
- q.resultReceived(new EndpointResult("op1", new Result.Detail(endpoint)), 0);
- assertThat(q.getPendingSize(), is(0));
- q.resultReceived(new EndpointResult("abc", new Result.Detail(endpoint)), 0);
- assertThat(q.getPendingSize(), is(0));
-
- assertThat(resultCount.get(), is(5));
-
- q.operationSent("op4", connection);
- assertThat(q.getPendingSize(), is(1));
- q.operationSent("op5", connection);
- assertThat(q.getPendingSize(), is(2));
-
- q.failPending(new RuntimeException());
-
- assertThat(resultCount.get(), is(7));
- }
-
-
- @Test
- public void testTimeout() throws InterruptedException {
- Endpoint endpoint = Endpoint.create("a");
- OperationProcessor mockAggregator = mock(OperationProcessor.class);
- CountDownLatch latch = new CountDownLatch(1);
- doAnswer(invocationOnMock -> {
- latch.countDown();
- return null;
- }).when(mockAggregator).resultReceived(any(), eq(0));
- EndpointResultQueue q = new EndpointResultQueue(
- mockAggregator, endpoint, 0, new ScheduledThreadPoolExecutor(1), 100L);
- q.operationSent("1234", new DryRunGatewayConnection(endpoint, Clock.systemUTC()));
- assert(latch.await(120, TimeUnit.SECONDS));
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java
deleted file mode 100644
index 780947b525a..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.communication;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.*;
-
-
-public class GatewayThrottlerTest {
-
- GatewayThrottler gatewayThrottler;
- long lastSleepValue = 0;
-
- @Before
- public void before() {
- gatewayThrottler = new GatewayThrottler(900) {
- @Override
- protected void sleepMs(long sleepTime) {
- lastSleepValue = sleepTime;
- }
- };
- }
-
- @Test
- public void noSleepOnNormalCase() {
- gatewayThrottler.handleCall(0);
- gatewayThrottler.handleCall(0);
- assertThat(lastSleepValue, is(0L));
- }
-
- @Test
- public void increastingSleepTime() {
- gatewayThrottler.handleCall(1);
- long sleepTime1 = lastSleepValue;
- gatewayThrottler.handleCall(1);
- long sleepTime2 = lastSleepValue;
- assertTrue(sleepTime1 > 0);
- assertTrue(sleepTime2 > sleepTime1);
- int x;
- // Check for max value of sleep time.
- for (x = 0 ; x < 10000; x++) {
- long prevSleepTime = lastSleepValue;
- gatewayThrottler.handleCall(1);
- if (prevSleepTime == lastSleepValue) break;
- }
- assertTrue(x < 5000);
- // Check that it goes down back to zero when no errors.
- for (x = 0 ; x < 10000; x++) {
- gatewayThrottler.handleCall(0);
- if (lastSleepValue == 0) break;
- }
- assertTrue(x < 5000);
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java
deleted file mode 100644
index 9984e43374a..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java
+++ /dev/null
@@ -1,175 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.communication;
-
-import com.yahoo.vespa.http.client.core.OperationProcessorTester;
-import com.yahoo.vespa.http.client.core.ServerResponseException;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.time.Duration;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-
-/**
- * TODO: Migrate IOThreadTests here.
- *
- * @author bratseth
- */
-public class IOThreadTest {
-
- @Test
- public void testSuccessfulWriting() {
- OperationProcessorTester tester = new OperationProcessorTester();
- assertEquals(0, tester.incomplete());
- assertEquals(0, tester.success());
- assertEquals(0, tester.failures());
- tester.send("doc1");
- tester.send("doc2");
- tester.send("doc3");
- assertEquals(3, tester.incomplete());
- assertEquals(0, tester.success());
- assertEquals(0, tester.failures());
- tester.tick(1); // connect
- assertEquals(3, tester.incomplete());
- tester.tick(1); // sync
- assertEquals(3, tester.incomplete());
- tester.tick(1); // process queue
- assertEquals(0, tester.incomplete());
- assertEquals(3, tester.success());
- assertEquals(0, tester.failures());
- }
-
- @Test
- public void testFatalExceptionOnHandshake() {
- OperationProcessorTester tester = new OperationProcessorTester();
- IOThread ioThread = tester.getSingleIOThread();
- DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection();
- firstConnection.throwOnHandshake(new ServerResponseException(403, "Not authorized"));
-
- tester.send("doc1");
- tester.send("doc2");
- tester.send("doc3");
- tester.tick(3);
- assertEquals(0, tester.incomplete());
- assertEquals(0, ioThread.resultQueue().getPendingSize());
- assertEquals(0, tester.success());
- assertEquals(3, tester.failures());
- }
-
- @Test
- public void testExceptionOnHandshake() {
- OperationProcessorTester tester = new OperationProcessorTester();
- IOThread ioThread = tester.getSingleIOThread();
- DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection();
- firstConnection.throwOnHandshake(new ServerResponseException(418, "I'm a teapot"));
-
- tester.send("doc1");
- tester.tick(3);
- assertEquals(1, tester.incomplete());
- assertEquals(0, ioThread.resultQueue().getPendingSize());
- assertEquals(0, tester.success());
- assertEquals("Awaiting retry", 0, tester.failures());
- }
-
- @Test
- public void testExceptionOnWrite() {
- OperationProcessorTester tester = new OperationProcessorTester();
- IOThread ioThread = tester.getSingleIOThread();
- DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection();
- firstConnection.throwOnWrite(new IOException("Test failure"));
-
- tester.send("doc1");
- tester.tick(3);
- assertEquals(1, tester.incomplete());
- assertEquals(0, ioThread.resultQueue().getPendingSize());
- assertEquals(0, tester.success());
- assertEquals("Awaiting retry since write exceptions is a transient failure",
- 0, tester.failures());
- }
-
- @Test
- public void testPollingOldConnections() {
- OperationProcessorTester tester = new OperationProcessorTester();
- tester.tick(3);
-
- IOThread ioThread = tester.getSingleIOThread();
- DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection();
- assertEquals(0, ioThread.oldConnections().size());
-
- firstConnection.hold(true);
- tester.send("doc1");
- tester.tick(1);
-
- tester.clock().advance(Duration.ofSeconds(31)); // Default connection ttl is 30
- tester.tick(3);
-
- assertEquals(1, ioThread.oldConnections().size());
- assertEquals(firstConnection, ioThread.oldConnections().get(0));
- assertNotSame(firstConnection, ioThread.currentConnection());
- assertEquals(31, firstConnection.lastPollTime().toEpochMilli() / 1000);
-
- // Check old connection poll pattern (exponential backoff)
- assertLastPollTimeWhenAdvancing(31, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
- assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester);
-
- tester.clock().advance(Duration.ofSeconds(200));
- tester.tick(1);
- assertEquals("Old connection is eventually removed", 0, ioThread.oldConnections().size());
- }
-
- @Test
- public void old_connections_are_closed_early_if_no_inflight_operations() {
- OperationProcessorTester tester = new OperationProcessorTester();
- tester.tick(3);
-
- IOThread ioThread = tester.getSingleIOThread();
- DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection();
- assertEquals(0, ioThread.oldConnections().size());
-
- firstConnection.hold(true); // do not send result for doc1 in next http response
- tester.send("doc1");
- tester.tick(1);
- tester.clock().advance(Duration.ofSeconds(31)); // Default connection TTL + 1
- tester.tick(1);
- assertEquals(1, ioThread.oldConnections().size());
-
- firstConnection.hold(false); // send result for both doc1 and doc2 in next http response
- tester.send("doc2");
- tester.tick(1);
- assertEquals(1, ioThread.oldConnections().size());
- tester.clock().advance(Duration.ofSeconds(2));
- tester.tick(3);
- assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester);
- assertEquals(0, ioThread.oldConnections().size());
- }
-
- private void assertLastPollTimeWhenAdvancing(int lastPollTimeSeconds,
- int advanceSeconds,
- DryRunGatewayConnection connection,
- OperationProcessorTester tester) {
- tester.clock().advance(Duration.ofSeconds(advanceSeconds));
- tester.tick(1);
- assertEquals(lastPollTimeSeconds, connection.lastPollTime().toEpochMilli() / 1000);
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java
deleted file mode 100644
index cd5da6b8589..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.operationProcessor;
-
-import com.yahoo.vespa.http.client.core.operationProcessor.ConcurrentDocumentOperationBlocker;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.*;
-
-public class ConcurrentDocumentOperationBlockerTest {
-
- final ConcurrentDocumentOperationBlocker blocker = new ConcurrentDocumentOperationBlocker();
- final CountDownLatch latch = new CountDownLatch(1);
-
- @Before
- public void setup() throws InterruptedException {
- blocker.setMaxConcurrency(2);
- blocker.startOperation();
- assertThat(blocker.availablePermits(), is(1));
- blocker.startOperation();
- }
-
- private void spawnThreadPushOperationThenCountDown() {
- new Thread(() -> {
- try {
- blocker.startOperation();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- latch.countDown();
- }).start();
- }
-
- @Test
- public void testBasics() throws InterruptedException {
- spawnThreadPushOperationThenCountDown();
- assertFalse(latch.await(10, TimeUnit.MILLISECONDS));
- blocker.operationDone();
- assertTrue(latch.await(120, TimeUnit.SECONDS));
- }
-
- @Test
- public void testResizeLarger() throws InterruptedException {
- spawnThreadPushOperationThenCountDown();
- assertFalse(latch.await(10, TimeUnit.MILLISECONDS));
- blocker.setMaxConcurrency(3);
- assertTrue(latch.await(120, TimeUnit.SECONDS));
- }
-
- @Test
- public void testResizeSmaller() throws InterruptedException {
- spawnThreadPushOperationThenCountDown();
- blocker.setMaxConcurrency(1);
- blocker.operationDone();
- assertFalse(latch.await(10, TimeUnit.MILLISECONDS));
- blocker.operationDone();
- assertTrue(latch.await(120, TimeUnit.SECONDS));
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java
deleted file mode 100644
index 82337cd4dcf..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java
+++ /dev/null
@@ -1,294 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.operationProcessor;
-
-import com.yahoo.vespa.http.client.ManualClock;
-import com.yahoo.vespa.http.client.core.ThrottlePolicy;
-import org.junit.Test;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.anyDouble;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class IncompleteResultsThrottlerTest {
-
- @Test
- public void simpleStaticQueueSizeTest() {
- IncompleteResultsThrottler incompleteResultsThrottler = new IncompleteResultsThrottler(2, 2, null, null);
- assertEquals(0, incompleteResultsThrottler.waitingThreads());
- incompleteResultsThrottler.operationStart();
- incompleteResultsThrottler.operationStart();
- assertEquals(2, incompleteResultsThrottler.waitingThreads());
- incompleteResultsThrottler.resultReady(true);
- assertEquals(1, incompleteResultsThrottler.waitingThreads());
- incompleteResultsThrottler.resultReady(true);
- assertEquals(0, incompleteResultsThrottler.waitingThreads());
- }
-
- /**
- * Simulate running requests.
- * @param clientCount number of parallel clients.
- * @param breakPoint how many requests the server should handle in parallel before it gets slower.
- * @param simulationTimeMs how many ms to simulate.
- * @return median queue length.
- */
- int getAverageQueue(int clientCount, int breakPoint, int simulationTimeMs) {
- ManualClock clock = new ManualClock(Instant.ofEpochMilli(0));
-
- ArrayList<IncompleteResultsThrottler> incompleteResultsThrottlers = new ArrayList<>();
-
- MockServer mockServer = new MockServer(breakPoint);
- for (int x = 0; x < clientCount; x++) {
- IncompleteResultsThrottler incompleteResultsThrottler =
- new IncompleteResultsThrottler(10, 50000, clock, new ThrottlePolicy());
- incompleteResultsThrottlers.add(incompleteResultsThrottler);
- }
- long sum = 0;
- long samples = 0;
-
- for (long time = 0; time < simulationTimeMs; time++) {
- // Fast forward, if we can. If all clients are blocked, we can move to the time when the server has a
- // request that is finished.
- boolean fastForward = true;
- for (int x = 0; x < clientCount; x++) {
- if (incompleteResultsThrottlers.get(x).availableCapacity() > 0 ) {
- fastForward = false;
- break;
- }
- }
- if (fastForward) {
- time = mockServer.nextRequestFinished();
- }
- clock.setInstant(Instant.ofEpochMilli(time));
- mockServer.moveTime(clock.instant().toEpochMilli());
- for (int y = 0; y < clientCount; y++) {
- // Fill up, but don't block as that would stop the simulation.
- while (incompleteResultsThrottlers.get(y).availableCapacity() > 0) {
- incompleteResultsThrottlers.get(y).operationStart();
- mockServer.newRequest(incompleteResultsThrottlers.get(y));
- }
- }
- // Don't take the first iterations into account as the system is eagerly learning.
- if (time > 60*1000) {
- sum += mockServer.messageDoneByTime.size();
- samples ++;
- }
- }
- return (int)(sum/samples);
- }
-
- private void testAndPrintVariousClientSizes(int breakPoint) {
- final int sampleRuns = 6;
- final int maxParallelClients = 4;
- final int minParallelClients = 1;
- final int simulationTimeMs = 400000;
- System.out.print("\nBreakpoint is " + breakPoint + ", average queue on server:");
- int[][] resultQueuesAverage = new int[maxParallelClients][sampleRuns];
- for (int clientNo = minParallelClients; clientNo <= maxParallelClients; clientNo++) {
- System.out.print("\nNow with " + clientNo + " parallel clients:");
- long sum = 0;
- for (int x = 0; x < sampleRuns; x++) {
- resultQueuesAverage[clientNo-minParallelClients][x] = getAverageQueue(1 + x, breakPoint, simulationTimeMs);
- System.out.print(" " + resultQueuesAverage[clientNo-minParallelClients][x]);
- sum += resultQueuesAverage[clientNo-minParallelClients][x];
- }
- System.out.print(" average is " + sum/sampleRuns);
- Arrays.sort(resultQueuesAverage[clientNo - minParallelClients]);
- int median = resultQueuesAverage[clientNo - minParallelClients][sampleRuns/2];
- System.out.print(" median is " + median);
- System.out.print(" min " + resultQueuesAverage[clientNo - minParallelClients][0]);
- System.out.print(" max " + resultQueuesAverage[clientNo - minParallelClients][sampleRuns - 1]);
- assertTrue(median < 2 * breakPoint + 200);
- assertTrue(median > breakPoint / 10);
- }
- }
-
- @Test
- public void testVariousBreakpoints() {
- testAndPrintVariousClientSizes(200);
- testAndPrintVariousClientSizes(1000);
- }
-
- List<Thread> threads = new ArrayList<>();
-
- private void postOperations(int count, final IncompleteResultsThrottler throttler) {
- for (int i = 0; i < count; i++) {
- Thread thread = new Thread(()->throttler.operationStart());
- thread.start();
- threads.add(thread);
- }
- }
-
- private void waitForThreads() throws InterruptedException {
- while(!threads.isEmpty()) {
- threads.remove(0).join();
- }
- }
-
- private void postSuccesses(int count, final IncompleteResultsThrottler throttler) {
- for (int i = 0; i < count; i++) {
- throttler.resultReady(true);
- }
- }
-
- private void moveToNextCycle(final IncompleteResultsThrottler throttler, ManualClock clock)
- throws InterruptedException {
- waitForThreads();
- // Enter an adaption phase, we don't care about this phase.
- clock.advance(Duration.ofMillis(throttler.phaseSizeMs));
- throttler.operationStart();
- throttler.resultReady(false);
- // Now enter the real next phase.
- clock.advance(Duration.ofMillis(throttler.phaseSizeMs));
- throttler.operationStart();
- throttler.resultReady(false);
- }
-
- @Test
- public void testInteractionWithPolicyByMockingPolicy() throws InterruptedException {
- ManualClock clock = new ManualClock(Instant.ofEpochMilli(0));
- final int MAX_SIZE = 1000;
- final int MORE_THAN_MAX_SIZE = MAX_SIZE + 20;
- final int SIZE_AFTER_CYCLE_FIRST = 30;
- final int SIZE_AFTER_CYCLE_SECOND = 5000;
- ThrottlePolicy policy = mock(ThrottlePolicy.class);
- IncompleteResultsThrottler incompleteResultsThrottler =
- new IncompleteResultsThrottler(2, MAX_SIZE, clock, policy);
- long bucketSizeMs = incompleteResultsThrottler.phaseSizeMs;
-
- // Cycle 1 - Algorithm has fixed value for max-in-flight: INITIAL_MAX_IN_FLIGHT_VALUE.
- // We post a few operations, not all finishing in this cycle. We explicitly do not fill the window
- // size to test the argument about any requests blocked.
- assertEquals(IncompleteResultsThrottler.INITIAL_MAX_IN_FLIGHT_VALUE,
- incompleteResultsThrottler.availableCapacity());
- postOperations(20, incompleteResultsThrottler);
- postSuccesses(15, incompleteResultsThrottler);
- moveToNextCycle(incompleteResultsThrottler, clock);
-
-
- // Cycle 2 - Algorithm has fixed value also for second iteration: SECOND_MAX_IN_FLIGHT_VALUE.
- // Test verifies that this value is used, and insert a value to be used for next phase SIZE_AFTER_CYCLE_FIRST.
- assertEquals("5 slots already taken earlier",
- IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE - 5,
- incompleteResultsThrottler.availableCapacity());
- postSuccesses(5, incompleteResultsThrottler);
- when(policy.calcNewMaxInFlight(
- anyDouble(), // Max performance change
- eq(5), //numOk
- eq(15), // previousNumOk
- eq(IncompleteResultsThrottler.INITIAL_MAX_IN_FLIGHT_VALUE), // previous size
- eq(IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE), // current size
- eq(false))) // is any request blocked, should be false since we only posted 20 docs.
- .thenReturn(SIZE_AFTER_CYCLE_FIRST);
- moveToNextCycle(incompleteResultsThrottler, clock);
-
- // Cycle 3 - Test that value set in previous phase is used. Now return a very large number.
- // However, this number should be cropped by the system (tested in next cycle).
- assertEquals(SIZE_AFTER_CYCLE_FIRST, incompleteResultsThrottler.availableCapacity());
- postOperations(MORE_THAN_MAX_SIZE, incompleteResultsThrottler);
- postSuccesses(MORE_THAN_MAX_SIZE, incompleteResultsThrottler);
- when(policy.calcNewMaxInFlight(
- anyDouble(), // Max performance change
- eq(MORE_THAN_MAX_SIZE), //numOk
- eq(5), // previousNumOk
- eq(IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE), // previous size
- eq(SIZE_AFTER_CYCLE_FIRST),// current size
- eq(true))) // is any request blocked, should be true since we posted MORE_THAN_MAX_SIZE docs.
- .thenReturn(SIZE_AFTER_CYCLE_SECOND);
- moveToNextCycle(incompleteResultsThrottler, clock);
-
- // Cycle 4 - Test that the large number from previous cycle is cropped and that max value is used instead.
- assertEquals(MAX_SIZE, incompleteResultsThrottler.availableCapacity());
- }
-
- private long inversesU(int size, int sweetSpot) {
- // Peak performance at sweetSPot.
- int distance = Math.abs(sweetSpot - size);
- return 1 + 20 * distance;
- }
-
- /**
- * A mock 'gateway' this is slower with more requests in-flight. It starts to become really much slower at
- * 'breakPoint' number of parallel requests.
- */
- class MockServer {
- final LinkedList<Tuple2<Long, IncompleteResultsThrottler> > messageDoneByTime = new LinkedList<>();
- final int breakPoint;
- final Random random = new Random();
- long time = 0;
-
- MockServer(int breakPoint) {
- this.breakPoint = breakPoint;
- }
-
- /**
- * Figures out when next processed data will be ready.
- * @return time in ms for next request to be finished.
- */
- long nextRequestFinished() {
- if (messageDoneByTime.isEmpty()) {
- return Integer.MAX_VALUE;
- }
- return messageDoneByTime.peek().first;
- }
-
- /**
- * Advance simulation time and call finished on any requests.
- * @param time to move to
- */
- void moveTime(long time) {
- this.time = time;
- while (!messageDoneByTime.isEmpty() && messageDoneByTime.peek().first <= time) {
- messageDoneByTime.pop().second.resultReady(true);
- }
- }
-
- /**
- * New request.
- * @param blocker do callback on blocker when request is done.
- */
- void newRequest(IncompleteResultsThrottler blocker) {
- long nextTime = (long)(20 + 0.1 * messageDoneByTime.size());
-
- if (messageDoneByTime.size() > breakPoint) {
- nextTime += (long) (40 + (random.nextDouble()) * 0.01 * messageDoneByTime.size()* messageDoneByTime.size());
- }
- nextTime += time + random.nextInt()%4;
- messageDoneByTime.push(new Tuple2<>(nextTime, blocker));
- }
- }
-
- private static class Tuple2<T1, T2> {
-
- public final T1 first;
- public final T2 second;
-
- public Tuple2(final T1 first, final T2 second) {
- this.first = first;
- this.second = second;
- }
-
- @Override
- public int hashCode() { throw new UnsupportedOperationException(); }
-
- @Override
- public boolean equals(final Object obj) { throw new UnsupportedOperationException(); }
-
- @Override
- public String toString() {
- return "Tuple2(" + first + ", " + second + ")";
- }
-
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java
deleted file mode 100644
index 7799f6089db..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java
+++ /dev/null
@@ -1,438 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.core.operationProcessor;
-
-import com.yahoo.vespa.http.client.Result;
-import com.yahoo.vespa.http.client.config.Cluster;
-import com.yahoo.vespa.http.client.config.ConnectionParams;
-import com.yahoo.vespa.http.client.config.Endpoint;
-import com.yahoo.vespa.http.client.config.SessionParams;
-import com.yahoo.vespa.http.client.core.Document;
-import com.yahoo.vespa.http.client.core.EndpointResult;
-import org.junit.Test;
-
-import java.time.Clock;
-import java.util.ArrayDeque;
-import java.util.Queue;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * @author Einar M R Rosenvinge
- */
-public class OperationProcessorTest {
-
- final Queue<Result> queue = new ArrayDeque<>();
- final Document doc1 = new Document("id:a:type::b", null, "data doc 1", null, Clock.systemUTC().instant());
- final Document doc1b = new Document("id:a:type::b", null, "data doc 1b", null, Clock.systemUTC().instant());
- final Document doc2 = new Document("id:a:type::b2", null, "data doc 2", null, Clock.systemUTC().instant());
- final Document doc3 = new Document("id:a:type::b3", null, "data doc 3", null, Clock.systemUTC().instant());
-
- @Test
- public void testBasic() {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .build();
-
- OperationProcessor q = new OperationProcessor(
- new IncompleteResultsThrottler(1000, 1000, null, null),
- (docId, documentResult) -> queue.add(documentResult),
- sessionParams, null, Clock.systemUTC());
-
-
- q.resultReceived(new EndpointResult("foo", new Result.Detail(null)), 0);
- assertEquals(0, queue.size());
-
-
- q.sendDocument(doc1);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("d"))), 3);
- assertEquals(1, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("e"))), 0);
- assertEquals(1, queue.size());
-
- //check a, b, c, d
- Result aggregated = queue.poll();
- assertEquals("id:a:type::b", aggregated.getDocumentId());
- assertEquals(4, aggregated.getDetails().size());
- assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname());
- assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname());
- assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname());
- assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname());
- assertEquals("data doc 1", aggregated.getDocumentDataAsCharSequence().toString());
-
- assertEquals(0, queue.size());
-
- q.sendDocument(doc2);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("d"))), 3);
- assertEquals(1, queue.size());
-
- q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("e"))), 0);
- assertEquals(1, queue.size());
-
- // check a, b, c, d
- aggregated = queue.poll();
- assertEquals("id:a:type::b2", aggregated.getDocumentId());
- assertEquals(4, aggregated.getDetails().size());
- assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname());
- assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname());
- assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname());
- assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname());
- assertEquals("data doc 2", aggregated.getDocumentDataAsCharSequence().toString());
-
- assertEquals(0, queue.size());
- }
-
- @Test
- public void testBlockingOfOperationsTwoEndpoints() {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .setConnectionParams(new ConnectionParams.Builder().build())
- .build();
- OperationProcessor operationProcessor = new OperationProcessor(
- new IncompleteResultsThrottler(1000, 1000, null, null),
- (docId, documentResult) -> queue.add(documentResult),
- sessionParams, null, Clock.systemUTC());
-
- operationProcessor.sendDocument(doc1);
- operationProcessor.sendDocument(doc1b);
-
- assertEquals(0, queue.size());
- // Only one operations should be in flight.
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0);
- assertEquals(0, queue.size());
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1);
- assertEquals(1, queue.size());
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0);
- assertEquals(1, queue.size());
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1);
- assertEquals(2, queue.size());
- assertEquals(0, operationProcessor.getIncompleteResultQueueSize());
- // This should have no effect.
- operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0);
- operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0);
- operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1);
- operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1);
- assertEquals(2, queue.size());
- }
-
- @Test
- public void testBlockingOfOperationsToSameDocIdWithTwoOperations() {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .setConnectionParams(new ConnectionParams.Builder().build())
- .build();
-
- OperationProcessor operationProcessor = new OperationProcessor(
- new IncompleteResultsThrottler(1000, 1000, null, null),
- (docId, documentResult) -> queue.add(documentResult),
- sessionParams, null, Clock.systemUTC());
-
- operationProcessor.sendDocument(doc1);
- operationProcessor.sendDocument(doc1b);
-
- assertEquals(0, queue.size());
- // Only one operations should be in flight.
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get());
- operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0);
- assertEquals(1, queue.size());
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get());
- operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0);
- assertEquals(2, queue.size());
- assertEquals(0, operationProcessor.getIncompleteResultQueueSize());
- assertFalse(operationProcessor.oldestIncompleteResultId().isPresent());
- // This should have no effect.
- operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0);
- operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0);
- assertEquals(2, queue.size());
- }
-
- @Test
- public void testBlockingOfOperationsToSameDocIdMany() {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .setConnectionParams(new ConnectionParams.Builder().build())
- .build();
-
- OperationProcessor operationProcessor = new OperationProcessor(
- new IncompleteResultsThrottler(1000, 1000, null, null),
- (docId, documentResult) -> queue.add(documentResult),
- sessionParams, null, Clock.systemUTC());
-
- Queue<Document> documentQueue = new ArrayDeque<>();
- for (int x = 0; x < 100; x++) {
- Document document = new Document("id:a:type::b", null, String.valueOf(x), null, Clock.systemUTC().instant());
- operationProcessor.sendDocument(document);
- documentQueue.add(document);
- }
-
- for (int x = 0; x < 100; x++) {
- assertEquals(x, queue.size());
- // Only one operations should be in flight.
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- Document document = documentQueue.poll();
- operationProcessor.resultReceived(new EndpointResult(document.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0);
- assertEquals(x+1, queue.size());
- if (x < 99) {
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- } else {
- assertEquals(0, operationProcessor.getIncompleteResultQueueSize());
- }
- }
- }
-
- @Test
- public void testMixOfBlockingAndNonBlocking() {
- Endpoint endpoint = Endpoint.create("localhost");
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(endpoint).build())
- .setConnectionParams(new ConnectionParams.Builder().build())
- .build();
-
- OperationProcessor operationProcessor = new OperationProcessor(
- new IncompleteResultsThrottler(1000, 1000, null, null),
- (docId, documentResult) -> queue.add(documentResult),
- sessionParams, null, Clock.systemUTC());
-
- operationProcessor.sendDocument(doc1);
- operationProcessor.sendDocument(doc1b); // Blocked
- operationProcessor.sendDocument(doc2);
- operationProcessor.sendDocument(doc3);
-
- assertEquals(0, queue.size());
- assertEquals(3, operationProcessor.getIncompleteResultQueueSize());
- assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get());
- // This should have no effect since it should not be sent.
- operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0);
- assertEquals(3, operationProcessor.getIncompleteResultQueueSize());
- assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get());
-
- operationProcessor.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(endpoint)), 0);
- assertEquals(2, operationProcessor.getIncompleteResultQueueSize());
- assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get());
- operationProcessor.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(endpoint)), 0);
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get());
- operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(endpoint)), 0);
- assertEquals(1, operationProcessor.getIncompleteResultQueueSize());
- assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get());
- operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0);
- assertEquals(0, operationProcessor.getIncompleteResultQueueSize());
- assertFalse(operationProcessor.oldestIncompleteResultId().isPresent());
- }
-
- @Test
- public void assertThatDuplicateResultsFromOneClusterWorks() {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .build();
-
- OperationProcessor q = new OperationProcessor(
- new IncompleteResultsThrottler(1000, 1000, null, null),
- (docId, documentResult) -> queue.add(documentResult),
- sessionParams, null, Clock.systemUTC());
-
- q.sendDocument(doc1);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 0);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 0);
- assertEquals(0, queue.size());
- }
-
- @Test
- public void testMultipleDuplicateDocIds() {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .build();
-
- OperationProcessor q = new OperationProcessor(
- new IncompleteResultsThrottler(1000, 1000, null, null),
- (docId, documentResult) -> queue.add(documentResult),
- sessionParams, null, Clock.systemUTC());
-
- q.sendDocument(doc1);
- assertEquals(0, queue.size());
- q.sendDocument(doc2);
- assertEquals(0, queue.size());
- q.sendDocument(doc3);
-
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1);
- assertEquals(0, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2);
- assertEquals(1, queue.size());
-
- q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0);
- assertEquals(1, queue.size());
-
- q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0);
- assertEquals(1, queue.size());
-
- q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1);
- assertEquals(1, queue.size());
-
- q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2);
- assertEquals(2, queue.size());
-
- q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2);
- assertEquals(2, queue.size());
-
- q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2);
- assertEquals(2, queue.size());
-
- q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1);
- assertEquals(3, queue.size());
-
- q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1);
- assertEquals(3, queue.size());
- assertEquals("data doc 1", queue.remove().getDocumentDataAsCharSequence().toString());
- assertEquals("data doc 2", queue.remove().getDocumentDataAsCharSequence().toString());
- assertEquals("data doc 3", queue.remove().getDocumentDataAsCharSequence().toString());
- }
-
- @Test
- public void testWaitBlocks() throws InterruptedException {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .build();
-
- OperationProcessor operationProcessor = new OperationProcessor(
- new IncompleteResultsThrottler(1, 1, null, null),
- (docId, documentResult) -> {},
- sessionParams, null, Clock.systemUTC());
-
- operationProcessor.sendDocument(doc1);
-
- final CountDownLatch started = new CountDownLatch(1);
- final CountDownLatch done = new CountDownLatch(1);
-
- Thread shouldWait = new Thread(()-> {
- started.countDown();
- operationProcessor.sendDocument(doc2);
- done.countDown();
- });
- shouldWait.start();
- started.await();
- // We want the test to pass fast so we only wait 40mS to see that it is blocking. This might lead to
- // some false positives, but that is ok.
- assertFalse(done.await(40, TimeUnit.MILLISECONDS));
- operationProcessor.resultReceived(
- new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("d"))), 0);
- assertTrue(done.await(120, TimeUnit.SECONDS));
-
- }
-
- @Test
- public void testSendsResponseToQueuedDocumentOnClose() throws InterruptedException {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .build();
-
- ScheduledThreadPoolExecutor executor = mock(ScheduledThreadPoolExecutor.class);
- when(executor.awaitTermination(anyLong(), any())).thenReturn(true);
-
- CountDownLatch countDownLatch = new CountDownLatch(3);
-
- OperationProcessor operationProcessor = new OperationProcessor(
- new IncompleteResultsThrottler(19, 19, null, null),
- (docId, documentResult) -> {
- countDownLatch.countDown();
- },
- sessionParams, executor, Clock.systemUTC());
-
- // Will fail due to bogus host name, but will be retried.
- operationProcessor.sendDocument(doc1);
- operationProcessor.sendDocument(doc2);
- operationProcessor.sendDocument(doc3);
-
- // Will create fail results.
- operationProcessor.close();
- countDownLatch.await();
- }
-
- @Test
- public void unknownHostThrowsExceptionAtConstructionTime() {
- try {
- SessionParams sessionParams = new SessionParams.Builder()
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build())
- .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("unknown.invalid")).build())
- .build();
- ScheduledThreadPoolExecutor executor = mock(ScheduledThreadPoolExecutor.class);
-
- CountDownLatch countDownLatch = new CountDownLatch(3);
-
- new OperationProcessor(
- new IncompleteResultsThrottler(19, 19, null, null),
- (docId, documentResult) -> {
- countDownLatch.countDown();
- },
- sessionParams, executor, Clock.systemUTC());
-
- fail("Expected exception");
- }
- catch (IllegalArgumentException e) {
- assertEquals("Unknown host: unknown.invalid:4080 ssl=false", e.getMessage());
- }
- }
-
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java
deleted file mode 100644
index bf4d4d0800b..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java
+++ /dev/null
@@ -1,417 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.handlers;
-
-import com.yahoo.vespa.http.client.core.Encoder;
-import com.yahoo.vespa.http.client.core.ErrorCode;
-import com.yahoo.vespa.http.client.core.Headers;
-import com.yahoo.vespa.http.client.core.OperationStatus;
-import org.eclipse.jetty.server.Request;
-import org.eclipse.jetty.server.handler.AbstractHandler;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a>
- * @since 5.1.27
- */
-public class V3MockParsingRequestHandler extends AbstractHandler {
- private final int responseCode;
- private volatile Scenario scenario;
- private final BlockingQueue<CountDownLatch> delayedRequests = new LinkedBlockingQueue<>();
- private final AtomicBoolean delayedResponseShouldBlock = new AtomicBoolean(true);
- public final AtomicBoolean badRequestScenarioShouldReturnBadRequest = new AtomicBoolean(false);
- private final String name;
- private static final AtomicInteger sessionIdGenerator = new AtomicInteger(0);
- private AtomicInteger internalCounter = new AtomicInteger(0);
-
- public enum Scenario {
- ALL_OK, RETURN_WRONG_SESSION_ID,
- DISCONNECT_IMMEDIATELY, DONT_ACCEPT_VERSION, RETURN_UNEXPECTED_VERSION,
- INTERNAL_SERVER_ERROR, COULD_NOT_FEED, MBUS_RETURNED_ERROR,
- NEVER_RETURN_ANY_RESULTS, DELAYED_RESPONSE, BAD_REQUEST, SERVER_ERROR_TWICE_THEN_OK,
- EXPECT_HIGHEST_PRIORITY_AND_TRACELEVEL_123, CONDITON_NOT_MET
- }
-
- public V3MockParsingRequestHandler() {
- this("", HttpServletResponse.SC_OK, Scenario.ALL_OK);
- }
-
- public V3MockParsingRequestHandler(String name) {
- this(name, HttpServletResponse.SC_OK, Scenario.ALL_OK);
- }
-
- public V3MockParsingRequestHandler(int responseCode) {
- this("", responseCode, Scenario.ALL_OK);
- }
-
- public V3MockParsingRequestHandler(int responseCode, Scenario scenario) {
- this("", responseCode, scenario);
- }
-
- public V3MockParsingRequestHandler(String name, int responseCode, Scenario scenario) {
- this.name = name;
- this.responseCode = responseCode;
- this.scenario = scenario;
- }
-
- public void setScenario(Scenario scenario) {
- this.scenario = scenario;
- }
-
- @Override
- public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
- System.err.println("Server " + name + " got request from: " + request.getHeader(Headers.SESSION_ID));
- switch (scenario) {
- case ALL_OK:
- allOk(baseRequest, request, response);
- break;
- case RETURN_WRONG_SESSION_ID:
- wrongSessionId(baseRequest, request, response);
- break;
- case DISCONNECT_IMMEDIATELY:
- disconnect(baseRequest, response);
- break;
- case DONT_ACCEPT_VERSION:
- dontAcceptVersion(baseRequest, request, response);
- break;
- case RETURN_UNEXPECTED_VERSION:
- unexpectedVersion(baseRequest, request, response);
- break;
- case INTERNAL_SERVER_ERROR:
- internalServerError(baseRequest, request, response);
- break;
- case COULD_NOT_FEED:
- couldNotFeed(baseRequest, request, response);
- break;
- case MBUS_RETURNED_ERROR:
- mbusReturnedError(baseRequest, request, response);
- break;
- case NEVER_RETURN_ANY_RESULTS:
- neverReturnAnyResults(baseRequest, request, response);
- break;
- case DELAYED_RESPONSE:
- delayedResponse(baseRequest, request, response);
- break;
- case BAD_REQUEST:
- badRequest(baseRequest, request, response);
- break;
- case SERVER_ERROR_TWICE_THEN_OK:
- int state = internalCounter.getAndIncrement();
- if (state >= 2) {
- allOk(baseRequest, request, response);
- } else {
- couldNotFeed(baseRequest, request, response);
- }
- break;
- case EXPECT_HIGHEST_PRIORITY_AND_TRACELEVEL_123:
- checkIfSessionThenHighPriorityAndTraceLevel123(request);
- allOk(baseRequest, request, response);
- break;
- case CONDITON_NOT_MET:
- conditionNotMetRequest(baseRequest, request, response);
- break;
- default:
- throw new IllegalArgumentException("Test scenario " + scenario + " not supported.");
- }
- }
-
- private void checkIfSessionThenHighPriorityAndTraceLevel123(HttpServletRequest request) {
- if (request.getHeader(Headers.SESSION_ID) != null) {
- assert (request.getHeader(Headers.PRIORITY).equals("HIGHEST"));
- assert (request.getHeader(Headers.TRACE_LEVEL).equals("123"));
- }
- }
-
- private void conditionNotMetRequest(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- String sessionId = getSessionId(request);
- setHeaders(response, sessionId);
- response.setStatus(responseCode);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- String operationId;
- while ((operationId = readOperationId(request.getInputStream())) != null) {
- long lengthToSkip = readByteLength(request.getInputStream());
- while (lengthToSkip > 0) {
- long skipped = request.getInputStream().skip(lengthToSkip);
- lengthToSkip -= skipped;
- }
- respondConditionNotMet(responseWriter, operationId);
- }
- closeChannel(responseWriter);
-
- }
- private void badRequest(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- if (badRequestScenarioShouldReturnBadRequest.get()) {
- response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- BufferedReader reader = new BufferedReader(new InputStreamReader(request.getInputStream()));
- while (reader.readLine() != null) {
- //consume input, not really needed?
- }
- reader.close();
- closeChannel(responseWriter);
- } else {
- allOk(baseRequest, request, response);
- }
- }
-
- private void delayedResponse(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- if (delayedResponseShouldBlock.get()) {
- CountDownLatch latch = new CountDownLatch(1);
- delayedRequests.add(latch);
- try {
- latch.await(120, TimeUnit.SECONDS); //wait "forever"
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- if (latch.getCount() != 0L) {
- throw new RuntimeException("Delayed request handler did not get poke()d.");
- }
- } else {
- }
- allOk(baseRequest, request, response);
- }
-
- public void poke() throws InterruptedException {
- CountDownLatch latch = delayedRequests.poll(10, TimeUnit.SECONDS);
- latch.countDown();
- }
-
- public void pokeAllAndUnblockFromNowOn() {
- delayedResponseShouldBlock.set(false);
- while (!delayedRequests.isEmpty()) {
- CountDownLatch latch = delayedRequests.remove();
- latch.countDown();
- }
- }
-
- private void allOk(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- String sessionId = getSessionId(request);
- setHeaders(response, sessionId);
- response.setStatus(responseCode);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- String operationId;
- while ((operationId = readOperationId(request.getInputStream())) != null) {
- long lengthToSkip = readByteLength(request.getInputStream());
- while (lengthToSkip > 0) {
- long skipped = request.getInputStream().skip(lengthToSkip);
- lengthToSkip -= skipped;
- }
- respondOK(responseWriter, operationId);
- }
- closeChannel(responseWriter);
- }
-
- private void wrongSessionId(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- String sessionId = generateMockSessionId();
- setHeaders(response, sessionId);
- response.setStatus(responseCode);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- String operationId;
- while ((operationId = readOperationId(request.getInputStream())) != null) {
- long lengthToSkip = readByteLength(request.getInputStream());
- while (lengthToSkip > 0) {
- long skipped = request.getInputStream().skip(lengthToSkip);
- lengthToSkip -= skipped;
- }
- respondOK(responseWriter, operationId);
- }
- closeChannel(responseWriter);
- }
-
- private void disconnect(Request baseRequest, HttpServletResponse response) throws IOException {
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- closeChannel(responseWriter);
- }
-
- private void dontAcceptVersion(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- String sessionId = getSessionId(request);
- setHeaders(response, sessionId);
- response.setStatus(Headers.HTTP_NOT_ACCEPTABLE);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- responseWriter.write("Go away, no such version.");
- responseWriter.flush();
- closeChannel(responseWriter);
- }
-
- private void unexpectedVersion(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- String sessionId = getSessionId(request);
- response.setHeader(Headers.SESSION_ID, sessionId);
- response.setHeader(Headers.VERSION, "12345678");
- response.setStatus(responseCode);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- String operationId;
- while ((operationId = readOperationId(request.getInputStream())) != null) {
- long lengthToSkip = readByteLength(request.getInputStream());
- while (lengthToSkip > 0) {
- long skipped = request.getInputStream().skip(lengthToSkip);
- lengthToSkip -= skipped;
- }
- respondOK(responseWriter, operationId);
- }
- closeChannel(responseWriter);
- }
-
- private void internalServerError(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- String sessionId = getSessionId(request);
- setHeaders(response, sessionId);
- response.setStatus(500);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- responseWriter.write("boom");
- responseWriter.flush();
- closeChannel(responseWriter);
- }
-
- private void couldNotFeed(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- String sessionId = getSessionId(request);
- setHeaders(response, sessionId);
- response.setStatus(responseCode);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- String operationId;
- while ((operationId = readOperationId(request.getInputStream())) != null) {
- long lengthToSkip = readByteLength(request.getInputStream());
- while (lengthToSkip > 0) {
- long skipped = request.getInputStream().skip(lengthToSkip);
- lengthToSkip -= skipped;
- }
- respondTransientFailed(responseWriter, operationId);
- }
- closeChannel(responseWriter);
- }
-
- private void mbusReturnedError(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- String sessionId = getSessionId(request);
- setHeaders(response, sessionId);
- response.setStatus(responseCode);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- String operationId;
- while ((operationId = readOperationId(request.getInputStream())) != null) {
- long lengthToSkip = readByteLength(request.getInputStream());
- while (lengthToSkip > 0) {
- long skipped = request.getInputStream().skip(lengthToSkip);
- lengthToSkip -= skipped;
- }
- respondFailedWithTransitiveErrorSeenFromClient(responseWriter, operationId);
- }
- closeChannel(responseWriter);
- }
-
- private void neverReturnAnyResults(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
- String sessionId = getSessionId(request);
- setHeaders(response, sessionId);
- response.setStatus(responseCode);
- baseRequest.setHandled(true);
- PrintWriter responseWriter = response.getWriter();
- BufferedReader reader = new BufferedReader(new InputStreamReader(request.getInputStream()));
- while (reader.readLine() != null) {
- //consume input, not really needed?
- }
- reader.close();
- closeChannel(responseWriter);
- }
-
- void closeChannel(PrintWriter responseWriter) {
- System.err.println("Mock server " + name + " closing channel.");
- responseWriter.close();
- }
-
- private String readOperationId(InputStream requestInputStream) throws IOException {
- StringBuilder idBuf = new StringBuilder(100);
- int c;
- while ((c = requestInputStream.read()) != -1) {
- if (c == 32) {
- break;
- }
- idBuf.append((char) c); //it's ASCII
- }
- if (c == -1) {
- return null;
- }
- return Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString();
- }
-
- private int readByteLength(InputStream requestInputStream) throws IOException {
- StringBuilder lenBuf = new StringBuilder(8);
- int c;
- while ((c = requestInputStream.read()) != -1) {
- if (c == 10) {
- break;
- }
- lenBuf.append((char) c); //it's ASCII
- }
- if (lenBuf.length() == 0) {
- throw new IllegalStateException("Operation length missing.");
- }
- return Integer.valueOf(lenBuf.toString(), 16);
- }
-
- private static void setHeaders(HttpServletResponse response, String sessionId) {
- response.setHeader(Headers.SESSION_ID, sessionId);
- response.setHeader(Headers.VERSION, "3");
- }
-
- private void respondFailed(PrintWriter responseWriter, String docId) {
- final OperationStatus operationStatus =
- new OperationStatus("mbus returned boom", docId, ErrorCode.ERROR, false, "trace");
- writeResponse(responseWriter, operationStatus);
- }
-
- private void respondTransientFailed(PrintWriter responseWriter, String docId) {
- final OperationStatus operationStatus = new OperationStatus(
- "Could not put", docId, ErrorCode.TRANSIENT_ERROR, false, "");
- writeResponse(responseWriter, operationStatus);
- }
-
- private void respondFailedWithTransitiveErrorSeenFromClient(PrintWriter responseWriter, String docId) {
- final OperationStatus operationStatus =
- new OperationStatus("NETWORK_ERROR", docId, ErrorCode.ERROR, false, "trace");
- writeResponse(responseWriter, operationStatus);
- }
-
- private void respondConditionNotMet(PrintWriter responseWriter, String docId) {
- final OperationStatus operationStatus =
- new OperationStatus("this is a test", docId, ErrorCode.ERROR, true, "trace");
- writeResponse(responseWriter, operationStatus);
- }
- private void respondOK(PrintWriter responseWriter, String docId) {
- final OperationStatus operationStatus = new OperationStatus("Doc fed", docId, ErrorCode.OK, false, "Trace message");
- writeResponse(responseWriter, operationStatus);
- }
-
- private void writeResponse(PrintWriter responseWriter,
- final OperationStatus operationStatus) {
- responseWriter.print(operationStatus.render());
- responseWriter.flush();
- System.err.println("Mock " + name + " server wrote: " + operationStatus.render());
- }
-
- private String getSessionId(HttpServletRequest request) {
- return request.getHeader(Headers.CLIENT_ID);
- }
-
- private String generateMockSessionId() {
- return String.valueOf(sessionIdGenerator.getAndIncrement());
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java
deleted file mode 100644
index 55f6af9ed32..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java
+++ /dev/null
@@ -1,210 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.runner;
-
-import com.yahoo.vespa.http.client.config.Cluster;
-import com.yahoo.vespa.http.client.config.FeedParams;
-import com.yahoo.vespa.http.client.config.SessionParams;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import static org.hamcrest.CoreMatchers.hasItem;
-import static org.hamcrest.CoreMatchers.nullValue;
-import static org.hamcrest.core.Is.is;
-import static org.hamcrest.core.IsNot.not;
-import static org.junit.Assert.assertThat;
-
-public class CommandLineArgumentsTest {
-
- private String[] asArray() {
- String[] array = new String[args.size()];
- args.toArray(array);
- return array;
- }
-
- private void add(String key, String value) {
- args.add("--" + key);
- args.add(value);
- }
-
- private void addMinimum() {
- add("host", "hostValue");
- }
-
- private ArrayList<String> args = new ArrayList<>();
-
- @Test
- public void testRequiredFlags() {
- assertThat(CommandLineArguments.build(asArray()), is(nullValue()));
- add("file", "fileValue");
- assertThat(CommandLineArguments.build(asArray()), is(nullValue()));
- args.clear();
- addMinimum();
- assertThat(CommandLineArguments.build(asArray()), is(not(nullValue())));
- }
-
- @Test
- public void testStreaming() {
- add("host", "hostValue");
- add("file", null); // Not yet implemented support for streaming
- assertThat(CommandLineArguments.build(asArray()), is(nullValue()));
- }
-
- @Test
- public void testBrokenFlags() {
- addMinimum();
- args.add("FOO");
- assertThat(CommandLineArguments.build(asArray()), is(nullValue()));
- }
-
- @Test
- public void testBadPriority() {
- addMinimum();
- add("priority", "non existing");
- assertThat(CommandLineArguments.build(asArray()), is(nullValue()));
- }
-
- @Test
- public void testOkPriority() {
- addMinimum();
- add("priority", "HIGHEST");
- assertThat(CommandLineArguments.build(asArray()).createSessionParams(false).getFeedParams().getPriority(),
- is("HIGHEST"));
- }
-
- @Test
- public void testDefaults() {
- addMinimum();
- CommandLineArguments arguments = CommandLineArguments.build(asArray());
- SessionParams params = arguments.createSessionParams(false /* use json */);
- assertThat(params.getClientQueueSize(), is(10000));
- assertThat(params.getThrottlerMinSize(), is(0));
- assertThat(params.getClusters().size(), is(1));
- assertThat(params.getClusters().get(0).getEndpoints().size(), is(1));
- assertThat(params.getClusters().get(0).getEndpoints().get(0).getHostname(), is("hostValue"));
- assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(4080));
- assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false));
- assertThat(params.getConnectionParams().getUseCompression(), is(false));
- assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(4));
- assertThat(params.getFeedParams().getRoute(), is("default"));
- assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.XML_UTF8));
- assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(180000L));
- assertThat(params.getFeedParams().getMaxInFlightRequests(), is(10000));
- assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(180000L));
- }
-
- @Test
- public void testAllImplementedFlags() {
- add("file", "fileValue.json");
- add("route", "routeValue");
- add("host", "hostValue");
- add("port", "1234");
- add("timeout", "2345");
- add("numPersistentConnectionsPerEndpoint", "7");
- args.add("--useCompression");
- args.add("--useDynamicThrottling");
- add("maxpending", "3456");
- args.add("--verbose");
- args.add("--useTls");
- add("header", "Header-Name: Header-Value");
- CommandLineArguments arguments = CommandLineArguments.build(asArray());
- SessionParams params = arguments.createSessionParams(true /* use json */);
- assertThat(params.getClientQueueSize(), is(3456));
- assertThat(params.getThrottlerMinSize(), is(10));
- assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(1234));
- assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(true));
- assertThat(params.getConnectionParams().getUseCompression(), is(true));
- assertThat(params.getConnectionParams().getHeaders().size(), is(1));
- assertThat(params.getFeedParams().getRoute(), is("routeValue"));
- assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.JSON_UTF8));
- assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(2345000L));
- assertThat(params.getFeedParams().getMaxInFlightRequests(), is(3456));
- assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(2345000L));
- assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(7));
- }
-
- @Test
- public void testAddingMultipleHttpHeaders() {
- add("host", "hostValue");
- String header1Name = "Header-Name-1";
- String header1Value = "Header-Value";
- add("header", header1Name + ": " + header1Value);
- String header2Name = "Header-Name-2";
- String header2Value = "Another-Header-Value";
- add("header", header2Name + ": " + header2Value);
-
- CommandLineArguments arguments = CommandLineArguments.build(asArray());
- SessionParams params = arguments.createSessionParams(true /* use json */);
-
- List<Map.Entry<String, String>> headers = new ArrayList<>(params.getConnectionParams().getHeaders());
- headers.sort(Comparator.comparing(Map.Entry::getKey));
-
- assertThat(headers.size(), is(2));
- Map.Entry<String, String> actualHeader1 = headers.get(0);
- assertThat(actualHeader1.getKey(), is(header1Name));
- assertThat(actualHeader1.getValue(), is(header1Value));
- Map.Entry<String, String> actualHeader2 = headers.get(1);
- assertThat(actualHeader2.getKey(), is(header2Name));
- assertThat(actualHeader2.getValue(), is(header2Value));
- }
-
- @Test
- public void testMultiHost() {
- add("file", "fileValue.json");
- add("port", "1234");
- add("host", "hostValue1,hostValue2, hostValue3");
- CommandLineArguments arguments = CommandLineArguments.build(asArray());
- SessionParams params = arguments.createSessionParams(true /* use json */);
- assertThat(params.getClusters().size(), is(3));
- final Set<String> hosts = new HashSet<>();
- for (Cluster cluster : params.getClusters()) {
- assertThat(cluster.getEndpoints().size(), is(1));
- hosts.add(cluster.getEndpoints().get(0).getHostname());
- assertThat(cluster.getEndpoints().get(0).getPort(), is(1234));
- }
- assertThat(hosts, hasItem("hostValue1"));
- assertThat(hosts, hasItem("hostValue2"));
- assertThat(hosts, hasItem("hostValue3"));
- }
-
- @Test
- public void testUseV3Protocol() {
- addMinimum();
- args.add("--useV3Protocol");
- CommandLineArguments arguments = CommandLineArguments.build(asArray());
- SessionParams params = arguments.createSessionParams(true /* use json */);
- }
-
- @Test
- public void testEndpoint() {
- add("endpoint", "http://myendpoint:1234");
- CommandLineArguments arguments = CommandLineArguments.build(asArray());
- SessionParams params = arguments.createSessionParams(true);
- assertThat(params.getClusters().get(0).getEndpoints().get(0).getHostname(), is("myendpoint"));
- assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(1234));
- assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false));
- }
-
- @Test
- public void testEndpointHttps() {
- add("endpoint", "https://myendpoint:1234");
- CommandLineArguments arguments = CommandLineArguments.build(asArray());
- SessionParams params = arguments.createSessionParams(true);
- assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(true));
- }
-
- @Test
- public void testEndpointAndHost() {
- add("host", "myhost");
- add("port", "2345");
- add("endpoint", "http://myendpoint:1234");
- CommandLineArguments arguments = CommandLineArguments.build(asArray());
- assertThat(arguments, is(nullValue())); // cannot have both endpoint and host
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java
deleted file mode 100644
index 9c77ed9a6d9..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.runner;
-
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Optional;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-
-/**
- * @author valerijf
- */
-public class FormatInputStreamTest {
- @Test(expected=IllegalArgumentException.class)
- public void testWithGarbageText() throws IOException {
- String streamString = "This is neither XML nor JSON!";
- InputStream jsonStream = getInputStreamOf(streamString);
- FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false);
- }
-
- @Test
- public void testWithFileInput() throws IOException {
- String fileString = "{\"format\": \"json\"}";
- File file = File.createTempFile("feeddata", "json");
- file.deleteOnExit();
- try (FileWriter writer = new FileWriter(file)) {
- writer.write(fileString);
- }
-
- FormatInputStream formatInputStream = new FormatInputStream(null, Optional.of(file.getAbsolutePath()), false);
- assertThat(fileString, is(convertStreamToString(formatInputStream.getInputStream())));
- assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON));
- }
-
- @Test
- public void testPreferenceFileOverStream() throws IOException {
- String streamString = "something entirely different";
- String fileString = "{\"format\": \"json\"}";
-
- InputStream jsonStream = getInputStreamOf(streamString);
- File file = File.createTempFile("feeddata", "json");
- file.deleteOnExit();
- try (FileWriter writer = new FileWriter(file)) {
- writer.write(fileString);
- }
-
- FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.of(file.getAbsolutePath()), false);
- assertThat(fileString, is(convertStreamToString(formatInputStream.getInputStream())));
- assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON));
- }
-
- @Test
- public void testSimpleJsonInputStream() throws IOException {
- String streamString = "{\"format\": \"json\"}";
- InputStream jsonStream = getInputStreamOf(streamString);
- FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false);
-
- assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream())));
- assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON));
- }
-
- @Test
- public void testSimpleXmlInputStream() throws IOException {
- String streamString = "<scope><tag>format</tag><value>xml</value></scope>";
- InputStream jsonStream = getInputStreamOf(streamString);
- FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false);
-
- assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream())));
- assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML));
- }
-
- @Test
- public void testSparselyFormattedXml() throws IOException {
- String streamString = " \t\t\n<scope>\n\n\n<tag>format</tag><value>xml</value></scope>";
- InputStream jsonStream = getInputStreamOf(streamString);
- FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false);
-
- assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream())));
- assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML));
- }
-
- @Test
- public void testAddRootToXml() throws IOException {
- String streamString = "some random text";
- InputStream textStream = getInputStreamOf(streamString);
- FormatInputStream formatInputStream = new FormatInputStream(textStream, Optional.empty(), true);
-
- assertThat("<vespafeed>" + streamString + "</vespafeed>",
- is(convertStreamToString(formatInputStream.getInputStream())));
- assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML));
- }
-
- private static String convertStreamToString(InputStream inputStream) throws IOException {
- StringBuilder builder = new StringBuilder();
- while (true) {
- int character = inputStream.read();
- if (character == -1) {
- inputStream.close();
- return builder.toString();
- }
- builder.append((char)character);
- }
- }
-
- private static InputStream getInputStreamOf(String text) {
- return new ByteArrayInputStream(text.getBytes());
- }
-}
diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java
deleted file mode 100644
index 0a5b3771958..00000000000
--- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java
+++ /dev/null
@@ -1,270 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.http.client.runner;
-
-import com.yahoo.vespa.http.client.FeedClient;
-import com.yahoo.vespa.http.client.core.JsonReader;
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static com.yahoo.test.json.JsonTestHelper.inputJson;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
-
-public class JsonReaderTest {
-
- private static String doc1_id = "id:unittest:testMapStringToArrayOfInt::whee";
-
- private static String doc1 = inputJson(
- "{",
- " 'update': '"+ doc1_id + "',",
- " 'fields': {",
- " 'actualMapStringToArrayOfInt': {",
- " 'assign': [",
- " { 'key': 'bamse', 'value': [ 2, 1, 3] }",
- " ]",
- " }",
- " }",
- "}");
-
- private static String doc2_id = "id:unittest:smoke::whee";
-
- private static String doc2 = inputJson(
- "{",
- " 'put': '" + doc2_id + "',",
- " 'fields': {",
- " 'something': 'smoketest',",
- " 'nalle': 'bamse'",
- " }",
- "}");
-
- private static String doc3 = inputJson(
- "{",
- " 'update': 'id:unittest:testarray::whee',",
- " 'fields': {",
- " 'actualarray': {",
- " 'add': [",
- " 'person naïve',",
- " 'another person'",
- " ]",
- " }",
- " }",
- "}");
-
- private static String doc4 = inputJson(
- "{",
- " 'remove': '" + doc2_id + "'",
- "}");
-
- private static String doc5_id = "id:unittest:smoking::wheels";
-
- private static String doc5 = inputJson(
- "{",
- " 'id': '" + doc5_id + "',",
- " 'fields': {",
- " 'something': 'smoketest',",
- " 'nalle': 'bamse'",
- " }",
- "}");
-
- private static class TestFeedClient implements FeedClient {
-
- public List<String> documentIds = new ArrayList<>();
- public List<CharSequence> datas = new ArrayList<>();
- public List<Object> contexts = new ArrayList<>();
-
- @Override
- public void stream(String documentId, CharSequence documentData) {
- stream(documentId, documentData, null);
- }
-
- @Override
- public void stream(String documentId, String operationId, CharSequence documentData, Object context) {
- documentIds.add(documentId);
- datas.add(documentData);
- contexts.add(context);
- }
-
- @Override
- public void close() { }
-
- @Override
- public String getStatsAsJson() { return null; }
- }
-
- final TestFeedClient session = new TestFeedClient();
- final AtomicInteger numSent = new AtomicInteger(0);
-
- @Test
- public void testReadNoDocument() throws Exception {
- InputStream inputStream = new ByteArrayInputStream(
- " ".getBytes(StandardCharsets.UTF_8));
- JsonReader.read(inputStream, session, numSent);
- inputStream.close();
- assertThat(session.documentIds.size(), is(0));
- }
-
- @Test
- public void testReadOneDocument() throws Exception {
- InputStream inputStream = new ByteArrayInputStream(
- ("["+ doc1 + "]" ).getBytes(StandardCharsets.UTF_8));
- JsonReader.read(inputStream, session, numSent);
- inputStream.close();
- assertThat(session.documentIds.size(), is(1));
- assertThat(session.documentIds.get(0), is(doc1_id));
- assertThat(session.datas.size(), is(1));
- assertThat(session.datas.get(0), is(doc1));
- }
-
- @Test
- public void testReadFourDocuments() throws Exception {
- InputStream inputStream = new ByteArrayInputStream(
- (" [ "+ doc1 + " , " + doc2 + ", " + doc3 + "," + doc4 + " ] ").getBytes(StandardCharsets.UTF_8));
- JsonReader.read(inputStream, session, numSent);
- inputStream.close();
- assertThat(session.documentIds.size(), is(4));
- assertThat(session.documentIds.get(0), is(doc1_id));
- assertThat(session.documentIds.get(1), is(doc2_id));
- assertThat(session.datas.size(), is(4));
- assertThat(session.datas.get(0), is(doc1));
- assertThat(session.datas.get(1).toString(), is(doc2));
- assertThat(session.datas.get(2).toString(), is(doc3));
- assertThat(session.datas.get(3).toString(), is(doc4));
- }
-
- @Test
- public void testDocWithIdAndNotPut() throws Exception {
- InputStream inputStream = new ByteArrayInputStream(
- (" [ "+ doc5 + " ] ").getBytes(StandardCharsets.UTF_8));
- JsonReader.read(inputStream, session, numSent);
- inputStream.close();
- assertThat(session.documentIds.size(), is(1));
- assertThat(session.documentIds.get(0), is(doc5_id));
- }
-
- @Test
- public void simpleMicroBenchmarkTest() throws Exception {
- StringBuilder stream = new StringBuilder();
- stream.append("[");
- int docsInStream = 15000;
- for (int x = 0; x < docsInStream -1; x++) {
- if (x % 10 == 0) {
- stream.append(doc1 + ", ");
- } else {
- // Add some randomness to the layout to trigger potential bugs in parsing.
- stream.append("{\"remove\": \"id:unittest:smoke::whee");
- for (int y = 0 ; y < x % 277 ; y++) {
- stream.append("X");
- }
- stream.append("\"}, ");
- }
- }
- stream.append(doc3);
- stream.append("]");
-
- InputStream inputStream = new ByteArrayInputStream(stream.toString().getBytes(StandardCharsets.UTF_8));
- long startTime = System.currentTimeMillis();
- JsonReader.read(inputStream, session, numSent);
- // At time of writing, it took about 200 ms on my mac.
- System.err.println("Run time is " + (System.currentTimeMillis() - startTime) + " ms");
- inputStream.close();
-
- // Verify that content is not rubbish.
- for (int x = 0; x < docsInStream - 1; x++) {
- if (x % 10 == 0) {
- assertThat(session.datas.get(x).toString(), is(doc1));
- assertThat(session.documentIds.get(x), is(doc1_id));
- }
- }
- assertThat(session.datas.get(docsInStream-1).toString(), is(doc3));
- assertThat(numSent.get(), is(docsInStream));
- }
-
- @Test(expected=RuntimeException.class)
- public void testBadJsonCommaAfterLastElement() {
- InputStream inputStream = new ByteArrayInputStream(
- ("["+ doc1 + ",]" ).getBytes(StandardCharsets.UTF_8));
- JsonReader.read(inputStream, session, numSent);
- }
-
- @Test(expected=RuntimeException.class)
- public void testTotalGarbage() {
- InputStream inputStream = new ByteArrayInputStream(("garbage" ).getBytes(StandardCharsets.UTF_8));
- JsonReader.read(inputStream, session, numSent);
- }
-
- @Test(expected=RuntimeException.class)
- public void testTwoDocIds() {
- InputStream inputStream = new ByteArrayInputStream(("[{\"remove\": \"id\", \"update\": \"id:\"}]"
- .getBytes(StandardCharsets.UTF_8)));
- JsonReader.read(inputStream, session, numSent);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void throwsOnMissingId() {
- InputStream inputStream = new ByteArrayInputStream(
- inputJson("[{'fields':{ 'something': 'smoketest', 'nalle': 'bamse' }}]").getBytes(StandardCharsets.UTF_8));
- JsonReader.read(inputStream, session, numSent);
- }
-
- @Test
- public void testFullDocument() throws Exception {
- InputStream inputStream = new ByteArrayInputStream((
- "[{\n" +
- " \"update\": \"id:foo:music:doc:foo:bar\",\n" +
- " \n" +
- " \"fields\": {\n" +
- " \"artist\": {\n" +
- " \"assign\": null" +
- " },\n" +
- " \n" +
- " \"albums\": {\n" +
- " \"assign\": [\n" +
- " \"Kramgoda laatar 4\",\n" +
- " \"Kramgoda laatar 5\",\n" +
- " \"Kramgoda laatar 6\"\n" +
- " ],\n" +
- " \"add\": [\n" +
- " \"Kramgoda laatar 7\",\n" +
- " \"Kramgoda laatar 8\"\n" +
- " ]\n" +
- " },\n" +
- " \"inceptionYear\": {\n" +
- " \"increment\": 4\n" +
- " },\n" +
- " \"concerts\": {\n" +
- " \"assign\": {\n" +
- " \"Torsby 1993\": 1000,\n" +
- " \"Uddevalla 2000\": 34\n" +
- " },\n" +
- " \"match\": {\n" +
- " \"element\": \"Sundsvall 1980\",\n" +
- " \"increment\": 5392\n" +
- " },\n" +
- " \"add\": {\n" +
- " \"Kiruna 1999\": 200,\n" +
- " \"Oslo 1998\": 2000\n" +
- " }\n" +
- " },\n" +
- " \"scores\": {\n" +
- " \"match\": {\n" +
- " \"element\": \"Sven Ingvars\",\n" +
- " \"match\": {\n" +
- " \"element\": 0,\n" +
- " \"increment\": 78\n" +
- " }\n" +
- " }\n" +
- " }\n" +
- " }\n" +
- "}]\n").getBytes(StandardCharsets.UTF_8));
- JsonReader.read(inputStream, session, numSent);
- inputStream.close();
- assertThat(session.documentIds.size(), is(1));
- assertThat(session.documentIds.get(0), is("id:foo:music:doc:foo:bar"));
- }
-}