diff options
author | Jon Bratseth <bratseth@gmail.com> | 2022-06-05 13:04:17 +0200 |
---|---|---|
committer | gjoranv <gv@verizonmedia.com> | 2022-06-08 11:45:30 +0200 |
commit | 8647f18a80c40c3fe98f7f68e0330683786eb50d (patch) | |
tree | 9cb2a4bb9658f40de10b6ded892b54f928ed6e6f /vespa-http-client/src/test/java | |
parent | eabb1d3195a1bd6d34def333854ad713bd6ac886 (diff) |
Remove vespa-http-client on Vespa 8
Diffstat (limited to 'vespa-http-client/src/test/java')
31 files changed, 0 insertions, 4209 deletions
diff --git a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java b/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java deleted file mode 100644 index 2ca2ea4d14a..00000000000 --- a/vespa-http-client/src/test/java/ExampleUsageFeedClientTest.java +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedClientFactory; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.Server; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.handlers.V3MockParsingRequestHandler; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Unit test that test documentation code. - * - * @author dybis - */ -public class ExampleUsageFeedClientTest { - - @Test - public void testExampleCode() { - Server serverA = - new Server(new V3MockParsingRequestHandler(200, V3MockParsingRequestHandler.Scenario.ALL_OK), 0); - Server serverB = - new Server(new V3MockParsingRequestHandler(200, V3MockParsingRequestHandler.Scenario.ALL_OK), 0); - - exampleCode("localhost", serverA.getPort(), "localhost", serverB.getPort()); - serverA.close(); - serverB.close(); - } - - private static CharSequence generateDocument(String docId) { - // Just a dummy example of an update document operation. - return "{\"update\": \""+ docId + "\"," - + " \"fields\": { \"actualMapStringToArrayOfInt\": {" - + " \"assign\": [" - + "{ \"key\": \"fooKey\", \"value\": [ 2,1, 3] }" - + "]}}}"; - } - - // Example usage of FeedClient - public static void exampleCode(String hostNameA, int portServerA, String hostNameB, int portServerB) { - boolean useSsl = false; - final SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameA, portServerA, useSsl)).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameB, portServerB, useSsl)).build()) - .setFeedParams(new FeedParams.Builder() - .setDataFormat(FeedParams.DataFormat.JSON_UTF8) - .build()) - .build(); - - AtomicInteger resultsReceived = new AtomicInteger(0); - AtomicInteger errorsReceived = new AtomicInteger(0); - - FeedClient feedClient = FeedClientFactory.create(sessionParams, new FeedClient.ResultCallback() { - @Override - public void onCompletion(String docId, Result documentResult) { - resultsReceived.incrementAndGet(); - if (! documentResult.getContext().equals(docId)) { - System.err.println("Context does not work as expected."); - errorsReceived.incrementAndGet(); - } - if (!documentResult.isSuccess()) { - System.err.println("Problems with docID " + docId + ":" + documentResult.toString()); - errorsReceived.incrementAndGet(); - } - } - }); - int sentCounter = 0; - List<String> docIds = Arrays.asList("1", "2", "3", "4"); - for (final String docId : docIds) { - CharSequence docData = generateDocument(docId); - feedClient.stream(docId, docData, docId); - sentCounter++; - } - feedClient.close(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java deleted file mode 100644 index ca956110a34..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.api.FeedClientImpl; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * Tests for the API, using dryrun option to mock gateway. - * - * @author dybis - */ -public class FeedClientTest { - - private final static String DOCID = "doc_id"; - - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder() - .addEndpoint(Endpoint.create("localhost")) - .build()) - .setConnectionParams(new ConnectionParams.Builder() - .setDryRun(true) - .build()) - .build(); - final AtomicInteger resultsReceived = new AtomicInteger(0); - - FeedClient.ResultCallback resultCallback = (docId, documentResult) -> { - assertTrue(documentResult.isSuccess()); - assertEquals(DOCID, docId); - resultsReceived.incrementAndGet(); - }; - - FeedClient feedClient = new FeedClientImpl(sessionParams, resultCallback, FeedClientFactory.createTimeoutExecutor(), Clock.systemUTC()); - - @Test - public void testStreamAndClose() { - feedClient.stream(DOCID, "blob"); - feedClient.close(); - assertEquals(1, resultsReceived.get()); - } - - @Test - public void testGetStatsAsJson() throws Exception { - feedClient.stream(DOCID, "blob"); - while (resultsReceived.get() == 0) {Thread.sleep(3); } - String stats = feedClient.getStatsAsJson(); - assertTrue(stats.contains("\"dryRun\":true")); - feedClient.close(); - } - - @Test - public void testFeedJson() { - InputStream stream = new ByteArrayInputStream((String.format("[{\"remove\": \"%s\"}]", DOCID) - .getBytes(StandardCharsets.UTF_8))); - AtomicInteger docCounter = new AtomicInteger(0); - FeedClient.feedJson(stream, feedClient, docCounter); - assertEquals(1, docCounter.get()); - feedClient.close(); - assertEquals(1, resultsReceived.get()); - } - - @Test - public void testFeedXml() { - InputStream stream = new ByteArrayInputStream((String.format( - "<document documenttype=\"music\" documentid=\"%s\">\n</document>\n", DOCID) - .getBytes(StandardCharsets.UTF_8))); - AtomicInteger docCounter = new AtomicInteger(0); - FeedClient.feedXml(stream, feedClient, docCounter); - assertEquals(1, docCounter.get()); - feedClient.close(); - assertEquals(1, resultsReceived.get()); - } - -} - diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java deleted file mode 100644 index 72edba7adb3..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/ManualClock.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.time.Clock; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.temporal.TemporalAmount; - -/** - * A clock which initially has the time of its creation but can only be advanced by calling advance - * - * @author bratseth - */ -public class ManualClock extends Clock { - - private Instant currentTime = Instant.now(); - - public ManualClock() {} - - public ManualClock(String utcIsoTime) { - this(at(utcIsoTime)); - } - - public ManualClock(Instant currentTime) { - this.currentTime = currentTime; - } - - public void advance(TemporalAmount temporal) { - currentTime = currentTime.plus(temporal); - } - - public void setInstant(Instant time) { - currentTime = time; - } - - @Override - public Instant instant() { return currentTime; } - - @Override - public ZoneId getZone() { return null; } - - @Override - public Clock withZone(ZoneId zone) { return null; } - - @Override - public long millis() { return currentTime.toEpochMilli(); } - - public static Instant at(String utcIsoTime) { - return LocalDateTime.parse(utcIsoTime, DateTimeFormatter.ISO_DATE_TIME).atZone(ZoneOffset.UTC).toInstant(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java deleted file mode 100644 index 078606274d6..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/Server.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.server.handler.AbstractHandler; - -/** - * @author Einar M R Rosenvinge - */ -public final class Server implements AutoCloseable { - - private final org.eclipse.jetty.server.Server server; - - public Server(AbstractHandler handler, int port) { - this.server = new org.eclipse.jetty.server.Server(port); - server.setHandler(handler); - try { - server.start(); - assert(server.isStarted()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void close() throws RuntimeException { - try { - server.stop(); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException("jetty server.stop() failed", e); - } - } - - public int getPort() { - return ((ServerConnector)server.getConnectors()[0]).getLocalPort(); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java deleted file mode 100644 index 5f4a0fdec9a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SimpleLoggerResultCallbackTest.java +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import org.junit.Test; - -import java.time.Instant; -import java.util.ArrayList; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class SimpleLoggerResultCallbackTest { - @Test - public void testAverageCalculation() { - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false); - Instant now = Instant.now(); - logger.newSamplingPeriod(now); - Result result = mock(Result.class); - when(result.isSuccess()).thenReturn(true); - // 3 documents in 0.2 secs --> 15 docs/sec - logger.onCompletion("1", result); - logger.onCompletion("1", result); - logger.onCompletion("1", result); - double rate = logger.newSamplingPeriod(now.plusMillis(200)).rate; - assertEquals(rate, 15., 0.1 /* delta */); - } - - @Test - public void testAverageCalculationExteremeValues() { - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false); - Instant now = Instant.now(); - logger.newSamplingPeriod(now); - // 0 duration, 0 documents - double rate = logger.newSamplingPeriod(now).rate; - assertEquals(rate, 0, 0.1 /* delta */); - } - - @Test - public void testOutput() { - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(3), 0, false); - Instant now = Instant.now(); - logger.newSamplingPeriod(now); - Result result = mock(Result.class); - when(result.isSuccess()).thenReturn(true); - // 3 documents in 0.2 secs --> 15 docs/sec - logger.onCompletion("1", result); - logger.onCompletion("1", result); - logger.onCompletion("1", result); - double rate = logger.newSamplingPeriod(now.plusMillis(200)).rate; - assertEquals(rate, 15., 0.1 /* delta */); - } - - private void verifyPrintout(boolean ignoreConditionNotMet) { - ArrayList<String> outputList = new ArrayList<>(); - - SimpleLoggerResultCallback logger = new SimpleLoggerResultCallback(new AtomicInteger(30), 0, ignoreConditionNotMet) { - @Override - protected void println(String output) { - outputList.add(output); - } - @Override - protected DocumentRate newSamplingPeriod(Instant now) { - return new DocumentRate(19999999.2342342366664); - } - }; - // 2 success, 1 failure - Result result = mock(Result.class); - when(result.isSuccess()).thenReturn(true); - when(result.isSuccessOrConditionNotMet()).thenReturn(true); - logger.onCompletion("1", result); - logger.onCompletion("1", result); - when(result.isSuccess()).thenReturn(false); - when(result.isSuccessOrConditionNotMet()).thenReturn(false); - when(result.toString()).thenReturn("fooError"); - logger.onCompletion("1", result); - logger.printProgress(); - assertThat(outputList.toString(), - containsString("Result received: 3 (1 failed so far, 30 sent, success rate 19999999.23 docs/sec).")); - assertThat(outputList.toString(), containsString("Failure: fooError")); - } - - @Test - public void testPrintout() { - verifyPrintout(false); - verifyPrintout(true); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java deleted file mode 100644 index 53b2b236d50..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/SyncFeedClientTest.java +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; - -import com.yahoo.vespa.http.client.SyncFeedClient.SyncOperation; -import com.yahoo.vespa.http.client.SyncFeedClient.SyncResult; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertNull; - -/** - * Tests the sync wrapper to the feed client - * - * @author bratseth - */ -public class SyncFeedClientTest { - - @Test - public void testFeedJson() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder() - .addEndpoint(Endpoint.create("localhost")) - .build()) - .setConnectionParams(new ConnectionParams.Builder() - .setDryRun(true) - .build()) - .build(); - SyncFeedClient feedClient = new SyncFeedClient(sessionParams); - - assertFeedSuccessful(feedClient); - assertFeedSuccessful(feedClient); // ensure the client can be reused - feedClient.close(); - } - - private void assertFeedSuccessful(SyncFeedClient feedClient) { - List<SyncOperation> operations = new ArrayList<>(); - - operations.add(new SyncOperation("id::test::1", - "{" + - " \"put\": \"id::test::1\"," + - " \"fields\": {" + - " \"title\": \"Title 1\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::2", - "{" + - " \"put\": \"id::test::2\"," + - " \"fields\": {" + - " \"title\": \"Title 2\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::3", - "{" + - " \"put\": \"id::test::3\"," + - " \"fields\": {" + - " \"title\": \"Title 3\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::3", // Another operation for the same document - "{" + - " \"put\": \"id::test::3\"," + - " \"fields\": {" + - " \"title\": \"Title 4\"" + - " }" + - "}")); - operations.add(new SyncOperation("id::test::4", - "{" + - " \"put\": \"id::test::4\"," + - " \"fields\": {" + - " \"title\": \"Title 4\"" + - " }" + - "}", "opId_4", null)); - operations.add(new SyncOperation("id::test::4", // Another operation for the same document - "{" + - " \"put\": \"id::test::4\"," + - " \"fields\": {" + - " \"title\": \"Title 44\"" + - " }" + - "}", "opId_44", null)); - - SyncResult result = feedClient.stream(operations); - - assertTrue(result.isSuccess()); - assertEquals(6, result.results().size()); - assertNull(result.exception()); - assertEquals("id::test::1", result.results().get(0).getDocumentId()); - assertEquals("id::test::2", result.results().get(1).getDocumentId()); - assertEquals("id::test::3", result.results().get(2).getDocumentId()); - assertEquals("id::test::3", result.results().get(3).getDocumentId()); - assertEquals("id::test::4", result.results().get(4).getDocumentId()); - assertEquals("id::test::4", result.results().get(5).getDocumentId()); - assertEquals("opId_4", result.results().get(4).getOperationId()); - assertEquals("opId_44", result.results().get(5).getOperationId()); - assertTrue(result.results().get(4).getDocumentDataAsCharSequence().toString().contains("\"Title 4\"")); - assertTrue(result.results().get(5).getDocumentDataAsCharSequence().toString().contains("\"Title 44\"")); - - result.results().forEach(r -> assertNotNull(r.getOperationId())); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java deleted file mode 100644 index 67b3fc7653d..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestDocument.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -/** -* @author Einar M R Rosenvinge -*/ -public class TestDocument { - private final String documentId; - private final byte[] contents; - - TestDocument(String documentId, byte[] contents) { - this.documentId = documentId; - this.contents = contents; - } - - public String getDocumentId() { - return documentId; - } - - public byte[] getContents() { - return contents; - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java deleted file mode 100644 index b27fbba3e96..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPInputStream; - -import static org.junit.Assert.assertNull; - -/** - * @author Einar M R Rosenvinge - */ -@SuppressWarnings("deprecation") -public class TestUtils { - - public static void writeDocuments(Session session, List<TestDocument> documents) throws IOException { - for (TestDocument document : documents) { - writeDocument(session, document); - } - } - - public static void writeDocument(Session session, TestDocument document) throws IOException { - OutputStream operation = session.stream(document.getDocumentId()); - operation.write(document.getContents()); - operation.close(); - } - - public static Map<String, Result> getResults(Session session, int num) throws InterruptedException { - Map<String, Result> results = new HashMap<>(); - for (int i = 0; i < num; i++) { - Result r = session.results().poll(120, TimeUnit.SECONDS); - if (r == null) { - String extraInfo = ""; - extraInfo = "stats=" + session.getStatsAsJson(); - throw new IllegalStateException("Did not receive result within timeout. (" + extraInfo + ") " + - "Results received: " + results.values()); - } - results.put(r.getDocumentId(), r); - } - assertNull(session.results().poll(100, TimeUnit.MILLISECONDS)); - return results; - } - - public static String zipStreamToString(InputStream inputStream) throws IOException { - GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream); - final StringBuilder rawContent = new StringBuilder(); - while (true) { - int x = gzipInputStream.read(); - if (x < 0) { - break; - } - rawContent.append((char) x); - } - return rawContent.toString(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java deleted file mode 100644 index f4c12974f6d..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ClusterTest.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class ClusterTest { - - @Test - public void testSimple() { - Cluster cluster = new Cluster.Builder().build(); - - assertThat(cluster.getEndpoints().size(), is(0)); - assertThat(cluster.getRoute(), nullValue()); - } - - @Test - public void testConfig() { - Cluster cluster = new Cluster.Builder() - .addEndpoint(Endpoint.create("a")) - .addEndpoint(Endpoint.create("b")) - .setRoute("blah") - .build(); - - assertThat(cluster.getEndpoints().size(), is(2)); - assertThat(cluster.getEndpoints().get(0).getHostname(), equalTo("a")); - assertThat(cluster.getEndpoints().get(1).getHostname(), equalTo("b")); - assertThat(cluster.getRoute(), equalTo("blah")); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java deleted file mode 100644 index 3fcc73e3cdc..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import javax.net.ssl.SSLContext; -import java.security.NoSuchAlgorithmException; -import java.util.Iterator; -import java.util.Map; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNull.nullValue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class ConnectionParamsTest { - - @Test - public void testDefaults() { - ConnectionParams params = new ConnectionParams.Builder().build(); - - assertThat(params.getHeaders().isEmpty(), is(true)); - assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(1)); - assertThat(params.getSslContext(), nullValue()); - } - - @Test - public void testSetters() throws NoSuchAlgorithmException { - ConnectionParams params = new ConnectionParams.Builder() - .addHeader("Foo", "Bar") - .addHeader("Foo", "Baz") - .addHeader("Banana", "Apple") - .setNumPersistentConnectionsPerEndpoint(2) - .setSslContext(SSLContext.getDefault()) - .build(); - - assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(2)); - - assertThat(params.getHeaders().isEmpty(), is(false)); - assertThat(params.getHeaders().size(), is(3)); - //Iteration order seems stable, let's keep it like this for now - Iterator<Map.Entry<String, String>> headers = params.getHeaders().iterator(); - Map.Entry<String, String> header1 = headers.next(); - assertThat(header1.getKey(), equalTo("Foo")); - assertThat(header1.getValue(), equalTo("Bar")); - Map.Entry<String, String> header2 = headers.next(); - assertThat(header2.getKey(), equalTo("Foo")); - assertThat(header2.getValue(), equalTo("Baz")); - Map.Entry<String, String> header3 = headers.next(); - assertThat(header3.getKey(), equalTo("Banana")); - assertThat(header3.getValue(), equalTo("Apple")); - } - - @Test - public void header_providers_are_registered() { - ConnectionParams.HeaderProvider dummyProvider1 = () -> "fooValue"; - ConnectionParams.HeaderProvider dummyProvider2 = () -> "barValue"; - ConnectionParams params = new ConnectionParams.Builder() - .addDynamicHeader("foo", dummyProvider1) - .addDynamicHeader("bar", dummyProvider2) - .build(); - Map<String, ConnectionParams.HeaderProvider> headerProviders = params.getDynamicHeaders(); - assertEquals(2, headerProviders.size()); - assertEquals(dummyProvider1, headerProviders.get("foo")); - assertEquals(dummyProvider2, headerProviders.get("bar")); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java deleted file mode 100644 index 415a7746b3a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/EndpointTest.java +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class EndpointTest { - - @Test - public void testBasic() { - Endpoint endpoint = Endpoint.create("foo"); - - assertThat(endpoint.getHostname(), equalTo("foo")); - assertThat(endpoint.getPort(), equalTo(4080)); - assertThat(endpoint.isUseSsl(), is(false)); - } - - @Test - public void testBasicWithHttpProtocolPrefix() { - Endpoint endpoint = Endpoint.create("http://foo"); - assertThat(endpoint.getHostname(), equalTo("foo")); - } - - @Test(expected = RuntimeException.class) - public void testBasicWithHttpsProtocolPrefix() { - Endpoint.create("https://foo"); - } - - @Test - public void testAdvanced() { - Endpoint endpoint = Endpoint.create("bar", 1234, true); - - assertThat(endpoint.getHostname(), equalTo("bar")); - assertThat(endpoint.getPort(), equalTo(1234)); - assertThat(endpoint.isUseSsl(), is(true)); - } - - @Test - public void testMethods() { - Endpoint a = Endpoint.create("a"); - Endpoint b = Endpoint.create("b"); - - assertThat(a, not(equalTo(b))); - assertThat(a.hashCode(), not(equalTo(b.hashCode()))); - - Endpoint a2 = Endpoint.create("a"); - - assertThat(a, equalTo(a2)); - assertThat(a.hashCode(), equalTo(a2.hashCode())); - - a.toString(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java deleted file mode 100644 index 82e0b4deac0..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/FeedParamsTest.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.config; - -import org.junit.Test; - -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.22 - */ -public class FeedParamsTest { - - @Test - public void testDefaults() { - FeedParams params = new FeedParams.Builder().build(); - - assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.JSON_UTF8)); - assertThat(params.getMaxChunkSizeBytes(), is(50 * 1024)); - assertThat(params.getRoute(), nullValue()); - assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(180L)); - assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(20L)); - } - - @Test - public void testConfig() { - FeedParams params = new FeedParams.Builder() - .setDataFormat(FeedParams.DataFormat.XML_UTF8) - .setMaxChunkSizeBytes(123) - .setRoute("abc") - .setClientTimeout(321, TimeUnit.SECONDS) - .build(); - - assertThat(params.getDataFormat(), equalTo(FeedParams.DataFormat.XML_UTF8)); - assertThat(params.getMaxChunkSizeBytes(), is(123)); - assertThat(params.getRoute(), equalTo("abc")); - assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(180L)); - assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(321L)); - - params = new FeedParams.Builder() - .setServerTimeout(333L, TimeUnit.SECONDS) - .setClientTimeout(222L, TimeUnit.SECONDS) - .build(); - - assertThat(params.getServerTimeout(TimeUnit.SECONDS), is(333L)); - assertThat(params.getClientTimeout(TimeUnit.SECONDS), is(222L)); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java deleted file mode 100644 index 8f707f2426e..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/DocumentTest.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import org.junit.Test; - -import java.nio.ByteBuffer; -import java.nio.ReadOnlyBufferException; -import java.time.Clock; - -import static org.junit.Assert.assertEquals; - -public class DocumentTest { - - @Test - public void simpleCaseOk() { - String docId = "doc id"; - String docContent = "foo"; - Document document = new Document(docId, docContent.getBytes(), null, Clock.systemUTC().instant()); - assertEquals(docId, document.getDocumentId()); - assertEquals(ByteBuffer.wrap(docContent.getBytes()), document.getData()); - assertEquals(docContent, document.getDataAsString().toString()); - // Make sure that data is not modified on retrieval. - assertEquals(docContent, document.getDataAsString().toString()); - assertEquals(ByteBuffer.wrap(docContent.getBytes()), document.getData()); - assertEquals(docId, document.getDocumentId()); - } - - @Test(expected = ReadOnlyBufferException.class) - public void notMutablePutTest() { - Document document = new Document("id", null, "data", null, Clock.systemUTC().instant()); - document.getData().put("a".getBytes()); - } - - @Test(expected = ReadOnlyBufferException.class) - public void notMutableCompactTest() { - Document document = new Document("id", null, "data", null, Clock.systemUTC().instant()); - document.getData().compact(); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java deleted file mode 100644 index aa926578a94..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/EncoderTestCase.java +++ /dev/null @@ -1,240 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import static org.junit.Assert.*; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Functional tests for encoding Encoder, i.e. encoding scheme only producing - * ASCII and never containing white space or control characters. - * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - */ -public class EncoderTestCase { - private final String basic = "abc123"; - private final String basic2 = "abc{20}123"; - private final String quotedIsLast = "abc{20}"; - private final String quotedIsLastDecoded = "abc "; - private final String basic2Decoded = "abc 123"; - private final String unterminated = "abc{33"; - private final String unterminated2 = "abc{"; - private final String emptyQuoted = "abc{}123"; - private final String outsideUnicode = "abc{7fffffff}"; - private final String noise = "abc{7fff{||\\ffff}"; - private final String fullAsciiEncoded = "{0}{1}{2}{3}{4}{5}{6}{7}{8}{9}" - + "{a}{b}{c}{d}{e}{f}{10}{11}{12}{13}{14}{15}{16}{17}" - + "{18}{19}{1a}{1b}{1c}{1d}{1e}{1f}{20}" - + "!\"#$%&'()*+,-./0123456789:;<=>?@" - + "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`" - + "abcdefghijklmnopqrstuvwxyz{7b}|{7d}~{7f}"; - private final int[] testCodepoints = { 0x0, '\n', ' ', 'a', '{', '}', 0x7f, 0x80, - 0x7ff, 0x800, 0xd7ff, 0xe000, 0xffff, 0x10000, 0x10ffff, 0x34, - 0x355, 0x2567, 0xfff, 0xe987, 0x100abc }; - private final String semiNastyEncoded = "{0}{a}{20}a{7b}{7d}{7f}{80}" - + "{7ff}{800}{d7ff}{e000}{ffff}{10000}{10ffff}4" - + "{355}{2567}{fff}{e987}{100abc}"; - private final String invalidUnicode = "abc\ud812"; - private final String invalidUnicodeEncoded = "abc{d812}"; - - StringBuilder s; - - @Before - public void setUp() { - s = new StringBuilder(); - } - - @After - public void tearDown() { - s = null; - } - - @Test - public final void testBasic() { - Encoder.encode(basic, s); - assertEquals(basic, s.toString()); - } - - @Test - public final void testBasic2() { - Encoder.encode(basic2Decoded, s); - assertEquals(basic2, s.toString()); - } - - @Test - public final void testEncodeAscii() { - Encoder.encode(fullAscii(), s); - assertEquals(fullAsciiEncoded, s.toString()); - } - - @Test - public final void testEncodeMixed() { - Encoder.encode(semiNasty(), s); - assertEquals(semiNastyEncoded, s.toString()); - } - - @Test - public final void testEncodeQuotedIsLast() { - Encoder.encode(quotedIsLastDecoded, s); - assertEquals(quotedIsLast, s.toString()); - } - - @Test - public final void testInvalidUnicode() { - Encoder.encode(invalidUnicode, s); - assertEquals(invalidUnicodeEncoded, s.toString()); - } - - - @Test - public final void testDecodeBasic() { - Encoder.decode(basic, s); - assertEquals(basic, s.toString()); - } - - @Test - public final void testDecodeBasic2() { - Encoder.decode(basic2, s); - assertEquals(basic2Decoded, s.toString()); - } - - @Test - public final void testDecodeAscii() { - Encoder.decode(fullAsciiEncoded, s); - assertEquals(fullAscii(), s.toString()); - } - - @Test - public final void testDecodeMixed() { - Encoder.decode(semiNastyEncoded, s); - assertEquals(semiNasty(), s.toString()); - } - - - - @Test - public final void testDecodeQuotedIsLast() { - Encoder.decode(quotedIsLast, s); - assertEquals(quotedIsLastDecoded, s.toString()); - } - - - @Test - public final void testDecodeUnterminated() { - try { - Encoder.decode(unterminated, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - @Test - public final void testDecodeUnterminated2() { - try { - Encoder.decode(unterminated2, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - - } - - @Test - public final void testEmptyQuoted() { - try { - Encoder.decode(emptyQuoted, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - @Test - public final void testOutsideUnicode() { - try { - Encoder.decode(outsideUnicode, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - - @Test - public final void testNoise() { - try { - Encoder.decode(noise, s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - @Test - public final void testIllegalInputCharacter() { - try { - Encoder.decode("abc\u00e5", s); - } catch (IllegalArgumentException e) { - return; - } - fail("Expected IllegalArgumentException"); - } - - - @Test - public final void testNoIllegalCharactersInOutputForAscii() { - Encoder.encode(fullAscii(), s); - checkNoNonAscii(s.toString()); - } - - @Test - public final void testNoIllegalCharactersInOutputForMixedInput() { - Encoder.encode(semiNasty(), s); - checkNoNonAscii(s.toString()); - } - - @Test - public final void testSymmetryAscii() { - StringBuilder forDecoding = new StringBuilder(); - Encoder.encode(fullAscii(), s); - Encoder.decode(s.toString(), forDecoding); - assertEquals(fullAscii(), forDecoding.toString()); - } - - @Test - public final void testSymmetryMixed() { - StringBuilder forDecoding = new StringBuilder(); - Encoder.encode(semiNasty(), s); - Encoder.decode(s.toString(), forDecoding); - assertEquals(semiNasty(), forDecoding.toString()); - } - - - private void checkNoNonAscii(String input) { - for (int i = 0; i < input.length(); ++i) { - char c = input.charAt(i); - if (c > '~' || c <= ' ') { - fail("Encoded data contained character ordinal " + Integer.toHexString(c)); - } - } - } - - private String fullAscii() { - StringBuilder s = new StringBuilder(); - for (int i = 0; i <= 0x7f; ++i) { - s.append((char) i); - } - return s.toString(); - } - - private String semiNasty() { - StringBuilder s = new StringBuilder(); - for (int i = 0; i < testCodepoints.length; ++i) { - s.append(Character.toChars(testCodepoints[i])); - } - return s.toString(); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java deleted file mode 100644 index 2e6efffdd2c..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/OperationProcessorTester.java +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.FeedEndpointException; -import com.yahoo.vespa.http.client.ManualClock; -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.communication.ClusterConnection; -import com.yahoo.vespa.http.client.core.communication.IOThread; -import com.yahoo.vespa.http.client.core.communication.IOThreadTest; -import com.yahoo.vespa.http.client.core.operationProcessor.IncompleteResultsThrottler; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; - -import java.time.Instant; -import java.util.List; -import java.util.concurrent.ScheduledThreadPoolExecutor; - -import static org.junit.Assert.assertEquals; - -/** - * Helper for testing with an operation processor - * - * @author bratseth - */ -public class OperationProcessorTester { - - private final Endpoint endpoint; - private final int clusterId = 0; - private final ManualClock clock; - private final TestResultCallback resultCallback; - private final OperationProcessor operationProcessor; - - public OperationProcessorTester() { - endpoint = Endpoint.create("test-endpoint"); - SessionParams.Builder params = new SessionParams.Builder(); - Cluster.Builder clusterParams = new Cluster.Builder(); - clusterParams.addEndpoint(endpoint); - params.addCluster(clusterParams.build()); - ConnectionParams.Builder connectionParams = new ConnectionParams.Builder(); - connectionParams.setDryRun(true); - connectionParams.setRunThreads(false); - params.setConnectionParams(connectionParams.build()); - - clock = new ManualClock(Instant.ofEpochMilli(0)); - resultCallback = new TestResultCallback(); - operationProcessor = new OperationProcessor(new IncompleteResultsThrottler(1, 100, clock, new ThrottlePolicy()), - resultCallback, - params.build(), - new ScheduledThreadPoolExecutor(1), - clock); - } - - public ManualClock clock() { return clock; } - - /** Asserts that this has but a single IOThread and returns it */ - public IOThread getSingleIOThread() { - assertEquals(1, clusterConnections().size()); - assertEquals(1, clusterConnections().get(0).ioThreads().size()); - return clusterConnections().get(0).ioThreads().get(0); - } - - /** Do n iteration of work in all io threads of this */ - public void tick(int n) { - for (int i = 0; i < n; i++) - for (ClusterConnection cluster : operationProcessor.clusters()) - for (IOThread thread : cluster.ioThreads()) - thread.tick(); - } - - public void send(String documentId) { - operationProcessor.sendDocument(new Document(documentId, documentId, "data of " + documentId, null, clock.instant())); - } - - public int incomplete() { - return operationProcessor.getIncompleteResultQueueSize(); - } - - public int success() { - return resultCallback.successes; - } - - public List<ClusterConnection> clusterConnections() { - return operationProcessor.clusters(); - } - - public int failures() { - return resultCallback.failures; - } - - public int endpointExceptions() { - return resultCallback.endpointExceptions; - } - - public Result lastResult() { - return resultCallback.lastResult; - } - - private static class TestResultCallback implements FeedClient.ResultCallback { - - private int successes = 0; - private int failures = 0; - private int endpointExceptions = 0; - private Result lastResult; - - @Override - public void onCompletion(String docId, Result documentResult) { - this.lastResult = documentResult; - if (documentResult.isSuccess()) - successes++; - else - failures++; - } - - @Override - public void onEndpointException(FeedEndpointException exception) { - endpointExceptions++; - } - - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java deleted file mode 100644 index d5a4d09fb8f..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/ThrottlePolicyTest.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import org.junit.Test; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -public class ThrottlePolicyTest { - - private final ThrottlePolicy throttlePolicy = new ThrottlePolicy(); - // Default values for tests. - private int numOk = 1000; - private int prevOk = 1000; - private int prevMax = 100; - private int max = 100; - private boolean queued = true; - private double dynamicFactor = 0.1; - - @Test - public void samePerformanceShouldTuneDown() { - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(95)); - } - - @Test - public void improvedPerformanceSameSizeShouldTuneDown() { - numOk += 200; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(89)); - } - - @Test - public void improvedPerformanceSmallerSizeTuneDownFurther() { - numOk += 200; - max = 70; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(63)); - } - - @Test - public void improvedPerformanceLargerSizeIncrease() { - numOk += 200; - max = 130; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(143)); - dynamicFactor = 100; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(156)); - } - - @Test - public void improvedPerformanceLargerSizeButQueuedFalse() { - numOk += 200; - max = 130; - queued = false; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(128)); - } - - @Test - public void lowerPerformanceSameSizeShouldIncrease() { - numOk -= 200; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(110)); - } - - @Test - public void lowerPerformanceSmallerSizeShouldIncreaseSize() { - numOk -= 200; - max = 30; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(33)); - dynamicFactor = 100; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(36)); - } - - @Test - public void lowerPerformanceLargerSizeTuneDownFurther() { - numOk -= 200; - max = 130; - assertThat(throttlePolicy.calcNewMaxInFlight(dynamicFactor, numOk, prevOk, prevMax, max, queued), is(116)); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java deleted file mode 100644 index e71cee0f4c2..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/XmlFeedReaderTest.java +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.yahoo.vespa.http.client.FeedClient; -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.xml.sax.SAXParseException; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.Is.is; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; - -public class XmlFeedReaderTest { - private final static String feedResource = "/vespacorpfeed-prod-sample.xml"; - - private final static String feedResource2 = "/xml-challenge.xml"; - private final static String feedResource3 = "/xml-challenge2.xml"; - private final static String feedResource4 = "/xml-challenge3.xml"; - - private static final XmlMapper xmlMapper = new XmlMapper(); - - private final String updateDocUpdate = - "<?xml version=\"1.0\"?>\n" + - "<vespafeed>\n" + - "<update documentid=\"id:banana:banana::complex\" documenttype=\"banana\">\n" + - " <add fieldpath=\"structarr\">\n" + - " <item>\n" + - " <bytearr>\n" + - " <item>30</item>\n" + - " <item>55</item>\n" + - " </bytearr>\n" + - " </item>\n" + - " </add>\n" + - "</update>\n" + - "</vespafeed>\n"; - - @Test - public void testReadUpdate() throws Exception { - InputStream stream = new ByteArrayInputStream(updateDocUpdate.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(1)); - } - - private final String updateDocRemove = - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + - "\n" + - "<vespafeed>\n" + - " <remove documentid=\"id:music:music::http://music.yahoo.com/Bob0/BestOf\" />\n" + - " <remove documentid=\"id:music:music::http://music.yahoo.com/Bob9/BestOf\" />\n" + - "</vespafeed>"; - - @Test - public void testReadRemove() throws Exception { - InputStream stream = new ByteArrayInputStream(updateDocRemove.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(2)); - } - - private final String insertDocOperation = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"+ - "<vespafeed>\n"+ - "\n"+ - " <document type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/bobdylan/BestOf\">\n"+ - " <title>Best of Bob Dylan</title>\n"+ - " </document>\n"+ - "\n"+ - " <document type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/metallica/BestOf\">\n"+ - " <title>Best of Metallica</title>\n"+ - " </document>\n"+ - "</vespafeed>"; - - @Test - public void testInsert() throws Exception { - InputStream stream = new ByteArrayInputStream(insertDocOperation.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(2)); - } - - private final String badperation = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"+ - "<vespafeed>\n"+ - " <badtag type=\"music\" documentid=\"id:music:music::http://music.yahoo.com/bobdylan/BestOf\">\n"+ - " <title>Best of Bob Dylan</title>\n"+ - " </badtag>\n"+ - "</vespafeed>"; - - @Test - public void testNonDocument() throws Exception { - InputStream stream = new ByteArrayInputStream(badperation.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(0)); - } - - @Test(expected=SAXParseException.class) - public void testGarbage() throws Exception { - InputStream stream = new ByteArrayInputStream("eehh".getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - XmlFeedReader.read(stream, feedClient, numSent); - } - - @Test - public void testEncoding() throws Exception { - InputStream stream = new ByteArrayInputStream("<?xml version=\"1.0\" encoding=\"utf8\"?><vespafeed><remove documentid=\"id:&\"/></vespafeed>" - .getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - - doAnswer(new Answer<Object>() { - public Object answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String docId = (String) args[0]; - CharSequence value = (CharSequence)args[1]; - assertThat(value.toString(), is("<remove documentid=\"id:&\"></remove>")); - assertThat(docId, is("id:&")); - return null; - } - }).when(feedClient).stream(anyString(), any()); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(1)); - } - - private final String characterDocs = "" + - "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + - "<!-- GENERATED VESPA-XML BY YSSTOXML -->\n" + - "<!-- USE ONLY FOR BATCH INDEXING -->\n" + - "<vespafeed>\n" + - " <document documenttype=\"simple\" documentid=\"id:test::&http://www.e.no/matprat\">\n" + - " <language><![CDATA[ja]]></language>\n" + - " <title><![CDATA[test document1]]></title>\n" + - " <description><![CDATA[Bjørnen' blåbær på øy nærheten.]]></description>\n" + - " <date>1091356845</date>\n" + - " <surl><![CDATA[http://www.eventyr.no/matprat]]></surl>\n" + - " </document>\n" + - "\n" + - "</vespafeed>\n"; - - @Test - public void testCharacterEndcoding() throws Exception { - InputStream stream = new ByteArrayInputStream(characterDocs.getBytes(StandardCharsets.UTF_8)); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - final AtomicBoolean success = new AtomicBoolean(false); - doAnswer(new Answer<Object>() { - public Object answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String docId = (String) args[0]; - CharSequence value = (CharSequence)args[1]; - assertThat(value.toString(), is( - "<document documenttype=\"simple\" documentid=\"id:test::&http://www.e.no/matprat\">\n" + - " <language><![CDATA[ja]]></language>\n" + - " <title><![CDATA[test document1]]></title>\n" + - " <description><![CDATA[Bjørnen' blåbær på øy nærheten.]]></description>\n" + - " <date>1091356845</date>\n" + - " <surl><![CDATA[http://www.eventyr.no/matprat]]></surl>\n" + - " </document>")); - success.set(true); - return null; - } - }).when(feedClient).stream(anyString(), any()); - XmlFeedReader.read(stream, feedClient, numSent); - assertThat(numSent.get(), is(1)); - assert(success.get()); - } - - @Test - public void testRealData() throws Exception { - InputStream inputStream = XmlFeedReaderTest.class.getResourceAsStream(feedResource); - BufferedInputStream bis = new BufferedInputStream(inputStream); - AtomicInteger numSent = new AtomicInteger(0); - FeedClient feedClient = mock(FeedClient.class); - - XmlFeedReader.read(bis, feedClient, numSent); - assertThat(numSent.get(), is(6)); - } - - private static class XmlTestFeedClient implements FeedClient { - - public List<String> documentIds = new ArrayList<>(); - public List<CharSequence> datas = new ArrayList<>(); - public List<Object> contexts = new ArrayList<>(); - - @Override - public void stream(String documentId, CharSequence documentData) { - stream(documentId, documentData, null); - } - - @Override - public void stream(String documentId, String operationId, CharSequence documentData, Object context) { - documentIds.add(documentId); - datas.add(documentData); - contexts.add(context); - } - - - @Override - public void close() { } - - @Override - public String getStatsAsJson() { return null; } - } - - // Only for xml with single doc. - private void verifyNoTransformationOfXml(String filename) throws Exception { - InputStream inputStream = XmlFeedReaderTest.class.getResourceAsStream(filename); - BufferedInputStream bis = new BufferedInputStream(inputStream); - AtomicInteger numSent = new AtomicInteger(0); - XmlTestFeedClient feedClient = new XmlTestFeedClient(); - XmlFeedReader.read(bis, feedClient, numSent); - assertThat(numSent.get(), is(1)); - String document = feedClient.datas.get(0).toString(); - - InputStream inputStream2 = XmlFeedReaderTest.class.getResourceAsStream(filename); - String rawXML = new java.util.Scanner(inputStream2, "UTF-8").useDelimiter("\\A").next(); - - JsonNode decodedDocument = xmlMapper.readTree(document); - JsonNode rawDocuments = xmlMapper.readTree(rawXML); - assertThat(decodedDocument, is(rawDocuments.get("document"))); - } - - @Test public void testCData() throws Exception { - verifyNoTransformationOfXml(feedResource2); - } - - @Test public void testPCData() throws Exception { - verifyNoTransformationOfXml(feedResource3); - } - - @Test public void testAposData() throws Exception { - verifyNoTransformationOfXml(feedResource4); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java deleted file mode 100644 index 283c1169440..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/api/FeedClientImplTest.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.api; - -import org.junit.Test; - -import java.time.Instant; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - -/** - * @author dybis - */ -public class FeedClientImplTest { - - int sleepValueMillis = 1; - - @Test - public void testCloseWaitTimeOldTimestamp() { - assertThat(FeedClientImpl.waitForOperations(Instant.now().minusSeconds(1000), sleepValueMillis, 10), is(false)); - } - - @Test - public void testCloseWaitTimeOutInFutureStillOperations() { - assertThat(FeedClientImpl.waitForOperations(Instant.now(), sleepValueMillis, 2000), is(true)); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java deleted file mode 100644 index 2597ddddd88..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnectionTest.java +++ /dev/null @@ -1,355 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.TestUtils; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import org.apache.http.Header; -import org.apache.http.HeaderElement; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.ParseException; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicHeader; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.stubbing.Answer; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.Clock; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class ApacheGatewayConnectionTest { - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - - @Test - public void testProtocolV3() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().build(); - List<Document> documents = new ArrayList<>(); - - String vespaDocContent = "Hello, I a JSON doc."; - String docId = "42"; - - // This is the fake server, takes header client ID and uses this as session Id. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header clientIdHeader = post.getFirstHeader(Headers.CLIENT_ID); - return httpResponse(clientIdHeader.getValue(), "3"); - }); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - documents.add(createDoc(docId, vespaDocContent, true)); - - apacheGatewayConnection.write(documents); - } - - @Test(expected=IllegalArgumentException.class) - public void testServerReturnsBadSessionInV3() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().build(); - - // This is the fake server, returns wrong session Id. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> httpResponse("Wrong Id from server", "3")); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - List<Document> documents = new ArrayList<>(); - apacheGatewayConnection.write(documents); - } - - @Test - public void testJsonDocumentHeader() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build(); - List<Document> documents = new ArrayList<>(); - - String vespaDocContent ="Hello, I a JSON doc."; - String docId = "42"; - - AtomicInteger requestsReceived = new AtomicInteger(0); - - // This is the fake server, checks that DATA_FORMAT header is set properly. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header header = post.getFirstHeader(Headers.DATA_FORMAT); - if (requestsReceived.incrementAndGet() == 1) { - // This is handshake, it is not json. - assert (header == null); - return httpResponse("clientId", "3"); - } - assertNotNull(header); - assertEquals(FeedParams.DataFormat.JSON_UTF8.name(), header.getValue()); - // Test is done. - return httpResponse("clientId", "3"); - }); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - documents.add(createDoc(docId, vespaDocContent, true)); - - apacheGatewayConnection.write(documents); - } - - @Test - public void testZipAndCreateEntity() throws IOException { - String testString = "Hello world"; - InputStream stream = new ByteArrayInputStream(testString.getBytes(StandardCharsets.UTF_8)); - // Send in test data to method. - InputStreamEntity inputStreamEntity = ApacheGatewayConnection.zipAndCreateEntity(stream); - // Verify zipped data by comparing unzipped data with test data. - String rawContent = TestUtils.zipStreamToString(inputStreamEntity.getContent()); - assertEquals(testString, rawContent); - } - - /** - * Mocks the HttpClient, and verifies that the compressed data is sent. - */ - @Test - public void testCompressedWriteOperations() throws Exception { - Endpoint endpoint = Endpoint.create("localhost", 666, false); - FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.XML_UTF8).build(); - String clusterSpecificRoute = ""; - ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build(); - List<Document> documents = new ArrayList<>(); - - String vespaDocContent ="Hello, I am the document data."; - String docId = "42"; - - Document doc = createDoc(docId, vespaDocContent, false); - - // When sending data on http client, check if it is compressed. If compressed, unzip, check result, - // and count down latch. - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header header = post.getFirstHeader("Content-Encoding"); - if (header != null && header.getValue().equals("gzip")) { - final String rawContent = TestUtils.zipStreamToString(post.getEntity().getContent()); - final String vespaHeaderText = "<vespafeed>\n"; - final String vespaFooterText = "</vespafeed>\n"; - - assertEquals(doc.getOperationId() + " 38\n" + vespaHeaderText + vespaDocContent + "\n" + vespaFooterText, - rawContent); - } - return httpResponse("clientId", "3"); - }); - - StatusLine statusLineMock = mock(StatusLine.class); - when(statusLineMock.getStatusCode()).thenReturn(200); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - endpoint, - feedParams, - clusterSpecificRoute, - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - documents.add(doc); - - apacheGatewayConnection.write(documents); - } - - @Test - public void dynamic_headers_are_added_to_the_response() throws IOException, ServerResponseException { - ConnectionParams.HeaderProvider headerProvider = mock(ConnectionParams.HeaderProvider.class); - when(headerProvider.getHeaderValue()) - .thenReturn("v1") - .thenReturn("v2") - .thenReturn("v3"); - - ConnectionParams connectionParams = new ConnectionParams.Builder() - .addDynamicHeader("foo", headerProvider) - .build(); - - AtomicInteger counter = new AtomicInteger(1); - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> { - Header[] fooHeader = post.getHeaders("foo"); - assertEquals(1, fooHeader.length); - assertEquals("foo", fooHeader[0].getName()); - assertEquals("v" + counter.getAndIncrement(), fooHeader[0].getValue()); - return httpResponse("clientId", "3"); - - }); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - Endpoint.create("localhost", 666, false), - new FeedParams.Builder().build(), - "", - connectionParams, - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - List<Document> documents = new ArrayList<>(); - documents.add(createDoc("42", "content", true)); - apacheGatewayConnection.write(documents); - apacheGatewayConnection.write(documents); - - verify(headerProvider, times(3)).getHeaderValue(); // 1x connect(), 2x writeOperations() - } - - @Test - public void detailed_error_message_is_extracted_from_error_responses_with_json() throws IOException, ServerResponseException { - String reasonPhrase = "Unauthorized"; - String errorMessage = "Invalid credentials"; - expectedException.expect(ServerResponseException.class); - expectedException.expectMessage(reasonPhrase + " - " + errorMessage); - - ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> createErrorHttpResponse(401, reasonPhrase, errorMessage)); - - ApacheGatewayConnection apacheGatewayConnection = - new ApacheGatewayConnection( - Endpoint.create("localhost", 666, false), - new FeedParams.Builder().build(), - "", - new ConnectionParams.Builder().build(), - mockFactory, - "clientId", - Clock.systemUTC()); - apacheGatewayConnection.connect(); - apacheGatewayConnection.handshake(); - - apacheGatewayConnection.write(Collections.singletonList(createDoc("42", "content", true))); - } - - private static ApacheGatewayConnection.HttpClientFactory mockHttpClientFactory(HttpExecuteMock httpExecuteMock) throws IOException { - ApacheGatewayConnection.HttpClientFactory mockFactory = - mock(ApacheGatewayConnection.HttpClientFactory.class); - CloseableHttpClient httpClientMock = mock(CloseableHttpClient.class); - when(mockFactory.createClient()).thenReturn(httpClientMock); - when(httpClientMock.execute(any())).thenAnswer((Answer) invocation -> { - Object[] args = invocation.getArguments(); - HttpPost post = (HttpPost) args[0]; - return httpExecuteMock.execute(post); - }); - return mockFactory; - } - - @FunctionalInterface private interface HttpExecuteMock { - HttpResponse execute(HttpPost httpPost) throws IOException; - } - - private Document createDoc(String docId, String content, boolean useJson) { - return new Document(docId, content.getBytes(), null, Clock.systemUTC().instant()); - } - - private void addMockedHeader(HttpResponse httpResponseMock, String name, String value, HeaderElement[] elements) { - Header header = new Header() { - @Override - public String getName() { - return name; - } - @Override - public String getValue() { - return value; - } - @Override - public HeaderElement[] getElements() throws ParseException { - return elements; - } - }; - when(httpResponseMock.getFirstHeader(name)).thenReturn(header); - } - - private HttpResponse httpResponse(String sessionIdInResult, String version) throws IOException { - CloseableHttpResponse httpResponseMock = mock(CloseableHttpResponse.class); - - StatusLine statusLineMock = mock(StatusLine.class); - when(httpResponseMock.getStatusLine()).thenReturn(statusLineMock); - when(statusLineMock.getStatusCode()).thenReturn(200); - - addMockedHeader(httpResponseMock, Headers.SESSION_ID, sessionIdInResult, null); - addMockedHeader(httpResponseMock, Headers.VERSION, version, null); - HeaderElement[] headerElements = new HeaderElement[1]; - headerElements[0] = mock(HeaderElement.class); - - final HttpEntity httpEntityMock = mock(HttpEntity.class); - when(httpResponseMock.getEntity()).thenReturn(httpEntityMock); - - final InputStream inputs = new ByteArrayInputStream("fake response data".getBytes()); - - when(httpEntityMock.getContent()).thenReturn(inputs); - return httpResponseMock; - } - - private static HttpResponse createErrorHttpResponse(int statusCode, String reasonPhrase, String message) throws IOException { - CloseableHttpResponse response = mock(CloseableHttpResponse.class); - - StatusLine statusLine = mock(StatusLine.class); - when(statusLine.getStatusCode()).thenReturn(statusCode); - when(statusLine.getReasonPhrase()).thenReturn(reasonPhrase); - when(response.getStatusLine()).thenReturn(statusLine); - - HttpEntity httpEntity = mock(HttpEntity.class); - when(httpEntity.getContentType()).thenReturn(new BasicHeader("Content-Type", "application/json")); - String json = String.format("{\"message\": \"%s\"}", message); - when(httpEntity.getContent()).thenReturn(new ByteArrayInputStream(json.getBytes())); - when(response.getEntity()).thenReturn(httpEntity); - return response; - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java deleted file mode 100644 index 9845768280e..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/ByteBufferInputStreamTest.java +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.communication.ByteBufferInputStream; -import org.junit.Test; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 - */ -public class ByteBufferInputStreamTest { - - private static ByteBuffer[] getAbcde() { - ByteBuffer[] buffers = new ByteBuffer[5]; - buffers[0] = ByteBuffer.wrap("a".getBytes(StandardCharsets.UTF_8)); - buffers[1] = ByteBuffer.wrap("b".getBytes(StandardCharsets.UTF_8)); - buffers[2] = ByteBuffer.wrap("c".getBytes(StandardCharsets.UTF_8)); - buffers[3] = ByteBuffer.wrap("d".getBytes(StandardCharsets.UTF_8)); - buffers[4] = ByteBuffer.wrap("e".getBytes(StandardCharsets.UTF_8)); - return buffers; - } - - @Test - public void requireThatExhaustedBufferWorks() throws IOException { - ByteBuffer[] buffers = getAbcde(); - buffers[2].get(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - - byte[] out = new byte[100]; - int pos = 0; - - final int GUARD = 1000; - int i; - for (i = 0; i < GUARD; i++) { - int r = in.read(); - if (r == -1) { - break; - } - out[pos] = (byte) (0xFF & r); - ++pos; - } - assertTrue(i < GUARD); - assertThat(pos, is(4)); - - String outString = new String(out, 0, pos, StandardCharsets.UTF_8); - assertThat(outString, equalTo("abde")); - - } - - @Test - public void requireThatBulkReadWorks() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - - byte[] out = new byte[100]; - int pos = 0; - - final int GUARD = 1000; - int i; - for (i = 0; i < GUARD; i++) { - int numReadNow; - if (i == 0) { - numReadNow = in.read(out); - } else { - numReadNow = in.read(out, pos, (out.length - pos)); - } - if (numReadNow == -1) { - break; - } - pos += numReadNow; - } - assertTrue(i < GUARD); - assertThat(pos, is(5)); - - String outString = new String(out, 0, pos, StandardCharsets.UTF_8); - assertThat(outString, equalTo("abcde")); - } - - @Test - public void requireThatSingleByteReadWorks() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - - byte[] out = new byte[100]; - int pos = 0; - - final int GUARD = 1000; - int i; - for (i = 0; i < GUARD; i++) { - int r = in.read(); - if (r == -1) { - break; - } - out[pos] = (byte) (0xFF & r); - ++pos; - } - assertTrue(i < GUARD); - assertThat(pos, is(5)); - - String outString = new String(out, 0, pos, StandardCharsets.UTF_8); - assertThat(outString, equalTo("abcde")); - } - - @Test - public void requireThatMarkIsNotSupported() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - assertThat(in.markSupported(), is(false)); - in.mark(0); //a no-op - } - - @Test(expected = IOException.class) - public void requireThatResetFails() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - in.reset(); - } - - @Test(expected = IOException.class) - public void requireThatSkipFails() throws IOException { - ByteBuffer[] buffers = getAbcde(); - ByteBufferInputStream in = new ByteBufferInputStream(buffers); - in.skip(1L); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java deleted file mode 100644 index 227f4a5239c..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/CloseableQTestCase.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.Document; -import org.junit.Test; - -import java.time.Clock; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class CloseableQTestCase { - - @Test - public void requestThatPutIsInterruptedOnClose() throws InterruptedException { - Clock clock = Clock.systemUTC(); - DocumentQueue q = new DocumentQueue(1, clock); - q.put(new Document("id", null, "data", null, clock.instant()), false); - Thread t = new Thread(new Runnable() { - @Override - public void run() { - try { - Thread.sleep(3000); - } catch (InterruptedException e) { - } - q.close(); - q.clear(); - } - }); - t.start(); - try { - q.put(new Document("id2", null, "data2", null, Clock.systemUTC().instant()), false); - fail("This shouldn't have worked."); - } catch (IllegalStateException ise) { - // ok! - } - try { - t.join(); - } catch (InterruptedException e) { - } - } - - @Test - public void requireThatSelfIsUnbounded() throws InterruptedException { - DocumentQueue q = new DocumentQueue(1, Clock.systemUTC()); - q.put(new Document("1", null, "data", null, Clock.systemUTC().instant()), true); - q.put(new Document("2", null, "data", null, Clock.systemUTC().instant()), true); - q.put(new Document("3", null, "data", null, Clock.systemUTC().instant()), true); - assertEquals(3, q.size()); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java deleted file mode 100644 index 78ccfed3dbc..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/EndpointResultQueueTest.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.core.EndpointResult; -import com.yahoo.vespa.http.client.core.operationProcessor.OperationProcessor; -import org.junit.Test; - -import java.time.Clock; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; - -/** - * @author Einar M R Rosenvinge - */ -public class EndpointResultQueueTest { - - @Test - public void testBasics() { - Endpoint endpoint = Endpoint.create("a"); - - GatewayConnection connection = new DryRunGatewayConnection(endpoint, Clock.systemUTC()); - OperationProcessor mockAggregator = mock(OperationProcessor.class); - final AtomicInteger resultCount = new AtomicInteger(0); - - doAnswer(invocationOnMock -> { - resultCount.getAndIncrement(); - return null; - }).when(mockAggregator).resultReceived(any(), eq(0)); - - EndpointResultQueue q = new EndpointResultQueue( - mockAggregator, endpoint, 0, new ScheduledThreadPoolExecutor(1), 100L * 1000L); - - q.operationSent("op1", connection); - assertThat(q.getPendingSize(), is(1)); - q.operationSent("op2", connection); - assertThat(q.getPendingSize(), is(2)); - q.operationSent("op3", connection); - assertThat(q.getPendingSize(), is(3)); - q.resultReceived(new EndpointResult("op1", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(2)); - q.resultReceived(new EndpointResult("op2", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(1)); - q.resultReceived(new EndpointResult("op3", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(0)); - q.resultReceived(new EndpointResult("op1", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(0)); - q.resultReceived(new EndpointResult("abc", new Result.Detail(endpoint)), 0); - assertThat(q.getPendingSize(), is(0)); - - assertThat(resultCount.get(), is(5)); - - q.operationSent("op4", connection); - assertThat(q.getPendingSize(), is(1)); - q.operationSent("op5", connection); - assertThat(q.getPendingSize(), is(2)); - - q.failPending(new RuntimeException()); - - assertThat(resultCount.get(), is(7)); - } - - - @Test - public void testTimeout() throws InterruptedException { - Endpoint endpoint = Endpoint.create("a"); - OperationProcessor mockAggregator = mock(OperationProcessor.class); - CountDownLatch latch = new CountDownLatch(1); - doAnswer(invocationOnMock -> { - latch.countDown(); - return null; - }).when(mockAggregator).resultReceived(any(), eq(0)); - EndpointResultQueue q = new EndpointResultQueue( - mockAggregator, endpoint, 0, new ScheduledThreadPoolExecutor(1), 100L); - q.operationSent("1234", new DryRunGatewayConnection(endpoint, Clock.systemUTC())); - assert(latch.await(120, TimeUnit.SECONDS)); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java deleted file mode 100644 index 780947b525a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/GatewayThrottlerTest.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import org.junit.Before; -import org.junit.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; - - -public class GatewayThrottlerTest { - - GatewayThrottler gatewayThrottler; - long lastSleepValue = 0; - - @Before - public void before() { - gatewayThrottler = new GatewayThrottler(900) { - @Override - protected void sleepMs(long sleepTime) { - lastSleepValue = sleepTime; - } - }; - } - - @Test - public void noSleepOnNormalCase() { - gatewayThrottler.handleCall(0); - gatewayThrottler.handleCall(0); - assertThat(lastSleepValue, is(0L)); - } - - @Test - public void increastingSleepTime() { - gatewayThrottler.handleCall(1); - long sleepTime1 = lastSleepValue; - gatewayThrottler.handleCall(1); - long sleepTime2 = lastSleepValue; - assertTrue(sleepTime1 > 0); - assertTrue(sleepTime2 > sleepTime1); - int x; - // Check for max value of sleep time. - for (x = 0 ; x < 10000; x++) { - long prevSleepTime = lastSleepValue; - gatewayThrottler.handleCall(1); - if (prevSleepTime == lastSleepValue) break; - } - assertTrue(x < 5000); - // Check that it goes down back to zero when no errors. - for (x = 0 ; x < 10000; x++) { - gatewayThrottler.handleCall(0); - if (lastSleepValue == 0) break; - } - assertTrue(x < 5000); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java deleted file mode 100644 index 9984e43374a..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/communication/IOThreadTest.java +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.communication; - -import com.yahoo.vespa.http.client.core.OperationProcessorTester; -import com.yahoo.vespa.http.client.core.ServerResponseException; -import org.junit.Test; - -import java.io.IOException; -import java.time.Duration; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; - -/** - * TODO: Migrate IOThreadTests here. - * - * @author bratseth - */ -public class IOThreadTest { - - @Test - public void testSuccessfulWriting() { - OperationProcessorTester tester = new OperationProcessorTester(); - assertEquals(0, tester.incomplete()); - assertEquals(0, tester.success()); - assertEquals(0, tester.failures()); - tester.send("doc1"); - tester.send("doc2"); - tester.send("doc3"); - assertEquals(3, tester.incomplete()); - assertEquals(0, tester.success()); - assertEquals(0, tester.failures()); - tester.tick(1); // connect - assertEquals(3, tester.incomplete()); - tester.tick(1); // sync - assertEquals(3, tester.incomplete()); - tester.tick(1); // process queue - assertEquals(0, tester.incomplete()); - assertEquals(3, tester.success()); - assertEquals(0, tester.failures()); - } - - @Test - public void testFatalExceptionOnHandshake() { - OperationProcessorTester tester = new OperationProcessorTester(); - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - firstConnection.throwOnHandshake(new ServerResponseException(403, "Not authorized")); - - tester.send("doc1"); - tester.send("doc2"); - tester.send("doc3"); - tester.tick(3); - assertEquals(0, tester.incomplete()); - assertEquals(0, ioThread.resultQueue().getPendingSize()); - assertEquals(0, tester.success()); - assertEquals(3, tester.failures()); - } - - @Test - public void testExceptionOnHandshake() { - OperationProcessorTester tester = new OperationProcessorTester(); - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - firstConnection.throwOnHandshake(new ServerResponseException(418, "I'm a teapot")); - - tester.send("doc1"); - tester.tick(3); - assertEquals(1, tester.incomplete()); - assertEquals(0, ioThread.resultQueue().getPendingSize()); - assertEquals(0, tester.success()); - assertEquals("Awaiting retry", 0, tester.failures()); - } - - @Test - public void testExceptionOnWrite() { - OperationProcessorTester tester = new OperationProcessorTester(); - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - firstConnection.throwOnWrite(new IOException("Test failure")); - - tester.send("doc1"); - tester.tick(3); - assertEquals(1, tester.incomplete()); - assertEquals(0, ioThread.resultQueue().getPendingSize()); - assertEquals(0, tester.success()); - assertEquals("Awaiting retry since write exceptions is a transient failure", - 0, tester.failures()); - } - - @Test - public void testPollingOldConnections() { - OperationProcessorTester tester = new OperationProcessorTester(); - tester.tick(3); - - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - assertEquals(0, ioThread.oldConnections().size()); - - firstConnection.hold(true); - tester.send("doc1"); - tester.tick(1); - - tester.clock().advance(Duration.ofSeconds(31)); // Default connection ttl is 30 - tester.tick(3); - - assertEquals(1, ioThread.oldConnections().size()); - assertEquals(firstConnection, ioThread.oldConnections().get(0)); - assertNotSame(firstConnection, ioThread.currentConnection()); - assertEquals(31, firstConnection.lastPollTime().toEpochMilli() / 1000); - - // Check old connection poll pattern (exponential backoff) - assertLastPollTimeWhenAdvancing(31, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(37, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - assertLastPollTimeWhenAdvancing(45, 1, firstConnection, tester); - - tester.clock().advance(Duration.ofSeconds(200)); - tester.tick(1); - assertEquals("Old connection is eventually removed", 0, ioThread.oldConnections().size()); - } - - @Test - public void old_connections_are_closed_early_if_no_inflight_operations() { - OperationProcessorTester tester = new OperationProcessorTester(); - tester.tick(3); - - IOThread ioThread = tester.getSingleIOThread(); - DryRunGatewayConnection firstConnection = (DryRunGatewayConnection)ioThread.currentConnection(); - assertEquals(0, ioThread.oldConnections().size()); - - firstConnection.hold(true); // do not send result for doc1 in next http response - tester.send("doc1"); - tester.tick(1); - tester.clock().advance(Duration.ofSeconds(31)); // Default connection TTL + 1 - tester.tick(1); - assertEquals(1, ioThread.oldConnections().size()); - - firstConnection.hold(false); // send result for both doc1 and doc2 in next http response - tester.send("doc2"); - tester.tick(1); - assertEquals(1, ioThread.oldConnections().size()); - tester.clock().advance(Duration.ofSeconds(2)); - tester.tick(3); - assertLastPollTimeWhenAdvancing(33, 1, firstConnection, tester); - assertEquals(0, ioThread.oldConnections().size()); - } - - private void assertLastPollTimeWhenAdvancing(int lastPollTimeSeconds, - int advanceSeconds, - DryRunGatewayConnection connection, - OperationProcessorTester tester) { - tester.clock().advance(Duration.ofSeconds(advanceSeconds)); - tester.tick(1); - assertEquals(lastPollTimeSeconds, connection.lastPollTime().toEpochMilli() / 1000); - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java deleted file mode 100644 index cd5da6b8589..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/ConcurrentDocumentOperationBlockerTest.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.core.operationProcessor.ConcurrentDocumentOperationBlocker; -import org.junit.Before; -import org.junit.Test; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; - -public class ConcurrentDocumentOperationBlockerTest { - - final ConcurrentDocumentOperationBlocker blocker = new ConcurrentDocumentOperationBlocker(); - final CountDownLatch latch = new CountDownLatch(1); - - @Before - public void setup() throws InterruptedException { - blocker.setMaxConcurrency(2); - blocker.startOperation(); - assertThat(blocker.availablePermits(), is(1)); - blocker.startOperation(); - } - - private void spawnThreadPushOperationThenCountDown() { - new Thread(() -> { - try { - blocker.startOperation(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - latch.countDown(); - }).start(); - } - - @Test - public void testBasics() throws InterruptedException { - spawnThreadPushOperationThenCountDown(); - assertFalse(latch.await(10, TimeUnit.MILLISECONDS)); - blocker.operationDone(); - assertTrue(latch.await(120, TimeUnit.SECONDS)); - } - - @Test - public void testResizeLarger() throws InterruptedException { - spawnThreadPushOperationThenCountDown(); - assertFalse(latch.await(10, TimeUnit.MILLISECONDS)); - blocker.setMaxConcurrency(3); - assertTrue(latch.await(120, TimeUnit.SECONDS)); - } - - @Test - public void testResizeSmaller() throws InterruptedException { - spawnThreadPushOperationThenCountDown(); - blocker.setMaxConcurrency(1); - blocker.operationDone(); - assertFalse(latch.await(10, TimeUnit.MILLISECONDS)); - blocker.operationDone(); - assertTrue(latch.await(120, TimeUnit.SECONDS)); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java deleted file mode 100644 index 82337cd4dcf..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/IncompleteResultsThrottlerTest.java +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.ManualClock; -import com.yahoo.vespa.http.client.core.ThrottlePolicy; -import org.junit.Test; - -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.Random; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyDouble; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class IncompleteResultsThrottlerTest { - - @Test - public void simpleStaticQueueSizeTest() { - IncompleteResultsThrottler incompleteResultsThrottler = new IncompleteResultsThrottler(2, 2, null, null); - assertEquals(0, incompleteResultsThrottler.waitingThreads()); - incompleteResultsThrottler.operationStart(); - incompleteResultsThrottler.operationStart(); - assertEquals(2, incompleteResultsThrottler.waitingThreads()); - incompleteResultsThrottler.resultReady(true); - assertEquals(1, incompleteResultsThrottler.waitingThreads()); - incompleteResultsThrottler.resultReady(true); - assertEquals(0, incompleteResultsThrottler.waitingThreads()); - } - - /** - * Simulate running requests. - * @param clientCount number of parallel clients. - * @param breakPoint how many requests the server should handle in parallel before it gets slower. - * @param simulationTimeMs how many ms to simulate. - * @return median queue length. - */ - int getAverageQueue(int clientCount, int breakPoint, int simulationTimeMs) { - ManualClock clock = new ManualClock(Instant.ofEpochMilli(0)); - - ArrayList<IncompleteResultsThrottler> incompleteResultsThrottlers = new ArrayList<>(); - - MockServer mockServer = new MockServer(breakPoint); - for (int x = 0; x < clientCount; x++) { - IncompleteResultsThrottler incompleteResultsThrottler = - new IncompleteResultsThrottler(10, 50000, clock, new ThrottlePolicy()); - incompleteResultsThrottlers.add(incompleteResultsThrottler); - } - long sum = 0; - long samples = 0; - - for (long time = 0; time < simulationTimeMs; time++) { - // Fast forward, if we can. If all clients are blocked, we can move to the time when the server has a - // request that is finished. - boolean fastForward = true; - for (int x = 0; x < clientCount; x++) { - if (incompleteResultsThrottlers.get(x).availableCapacity() > 0 ) { - fastForward = false; - break; - } - } - if (fastForward) { - time = mockServer.nextRequestFinished(); - } - clock.setInstant(Instant.ofEpochMilli(time)); - mockServer.moveTime(clock.instant().toEpochMilli()); - for (int y = 0; y < clientCount; y++) { - // Fill up, but don't block as that would stop the simulation. - while (incompleteResultsThrottlers.get(y).availableCapacity() > 0) { - incompleteResultsThrottlers.get(y).operationStart(); - mockServer.newRequest(incompleteResultsThrottlers.get(y)); - } - } - // Don't take the first iterations into account as the system is eagerly learning. - if (time > 60*1000) { - sum += mockServer.messageDoneByTime.size(); - samples ++; - } - } - return (int)(sum/samples); - } - - private void testAndPrintVariousClientSizes(int breakPoint) { - final int sampleRuns = 6; - final int maxParallelClients = 4; - final int minParallelClients = 1; - final int simulationTimeMs = 400000; - System.out.print("\nBreakpoint is " + breakPoint + ", average queue on server:"); - int[][] resultQueuesAverage = new int[maxParallelClients][sampleRuns]; - for (int clientNo = minParallelClients; clientNo <= maxParallelClients; clientNo++) { - System.out.print("\nNow with " + clientNo + " parallel clients:"); - long sum = 0; - for (int x = 0; x < sampleRuns; x++) { - resultQueuesAverage[clientNo-minParallelClients][x] = getAverageQueue(1 + x, breakPoint, simulationTimeMs); - System.out.print(" " + resultQueuesAverage[clientNo-minParallelClients][x]); - sum += resultQueuesAverage[clientNo-minParallelClients][x]; - } - System.out.print(" average is " + sum/sampleRuns); - Arrays.sort(resultQueuesAverage[clientNo - minParallelClients]); - int median = resultQueuesAverage[clientNo - minParallelClients][sampleRuns/2]; - System.out.print(" median is " + median); - System.out.print(" min " + resultQueuesAverage[clientNo - minParallelClients][0]); - System.out.print(" max " + resultQueuesAverage[clientNo - minParallelClients][sampleRuns - 1]); - assertTrue(median < 2 * breakPoint + 200); - assertTrue(median > breakPoint / 10); - } - } - - @Test - public void testVariousBreakpoints() { - testAndPrintVariousClientSizes(200); - testAndPrintVariousClientSizes(1000); - } - - List<Thread> threads = new ArrayList<>(); - - private void postOperations(int count, final IncompleteResultsThrottler throttler) { - for (int i = 0; i < count; i++) { - Thread thread = new Thread(()->throttler.operationStart()); - thread.start(); - threads.add(thread); - } - } - - private void waitForThreads() throws InterruptedException { - while(!threads.isEmpty()) { - threads.remove(0).join(); - } - } - - private void postSuccesses(int count, final IncompleteResultsThrottler throttler) { - for (int i = 0; i < count; i++) { - throttler.resultReady(true); - } - } - - private void moveToNextCycle(final IncompleteResultsThrottler throttler, ManualClock clock) - throws InterruptedException { - waitForThreads(); - // Enter an adaption phase, we don't care about this phase. - clock.advance(Duration.ofMillis(throttler.phaseSizeMs)); - throttler.operationStart(); - throttler.resultReady(false); - // Now enter the real next phase. - clock.advance(Duration.ofMillis(throttler.phaseSizeMs)); - throttler.operationStart(); - throttler.resultReady(false); - } - - @Test - public void testInteractionWithPolicyByMockingPolicy() throws InterruptedException { - ManualClock clock = new ManualClock(Instant.ofEpochMilli(0)); - final int MAX_SIZE = 1000; - final int MORE_THAN_MAX_SIZE = MAX_SIZE + 20; - final int SIZE_AFTER_CYCLE_FIRST = 30; - final int SIZE_AFTER_CYCLE_SECOND = 5000; - ThrottlePolicy policy = mock(ThrottlePolicy.class); - IncompleteResultsThrottler incompleteResultsThrottler = - new IncompleteResultsThrottler(2, MAX_SIZE, clock, policy); - long bucketSizeMs = incompleteResultsThrottler.phaseSizeMs; - - // Cycle 1 - Algorithm has fixed value for max-in-flight: INITIAL_MAX_IN_FLIGHT_VALUE. - // We post a few operations, not all finishing in this cycle. We explicitly do not fill the window - // size to test the argument about any requests blocked. - assertEquals(IncompleteResultsThrottler.INITIAL_MAX_IN_FLIGHT_VALUE, - incompleteResultsThrottler.availableCapacity()); - postOperations(20, incompleteResultsThrottler); - postSuccesses(15, incompleteResultsThrottler); - moveToNextCycle(incompleteResultsThrottler, clock); - - - // Cycle 2 - Algorithm has fixed value also for second iteration: SECOND_MAX_IN_FLIGHT_VALUE. - // Test verifies that this value is used, and insert a value to be used for next phase SIZE_AFTER_CYCLE_FIRST. - assertEquals("5 slots already taken earlier", - IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE - 5, - incompleteResultsThrottler.availableCapacity()); - postSuccesses(5, incompleteResultsThrottler); - when(policy.calcNewMaxInFlight( - anyDouble(), // Max performance change - eq(5), //numOk - eq(15), // previousNumOk - eq(IncompleteResultsThrottler.INITIAL_MAX_IN_FLIGHT_VALUE), // previous size - eq(IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE), // current size - eq(false))) // is any request blocked, should be false since we only posted 20 docs. - .thenReturn(SIZE_AFTER_CYCLE_FIRST); - moveToNextCycle(incompleteResultsThrottler, clock); - - // Cycle 3 - Test that value set in previous phase is used. Now return a very large number. - // However, this number should be cropped by the system (tested in next cycle). - assertEquals(SIZE_AFTER_CYCLE_FIRST, incompleteResultsThrottler.availableCapacity()); - postOperations(MORE_THAN_MAX_SIZE, incompleteResultsThrottler); - postSuccesses(MORE_THAN_MAX_SIZE, incompleteResultsThrottler); - when(policy.calcNewMaxInFlight( - anyDouble(), // Max performance change - eq(MORE_THAN_MAX_SIZE), //numOk - eq(5), // previousNumOk - eq(IncompleteResultsThrottler.SECOND_MAX_IN_FLIGHT_VALUE), // previous size - eq(SIZE_AFTER_CYCLE_FIRST),// current size - eq(true))) // is any request blocked, should be true since we posted MORE_THAN_MAX_SIZE docs. - .thenReturn(SIZE_AFTER_CYCLE_SECOND); - moveToNextCycle(incompleteResultsThrottler, clock); - - // Cycle 4 - Test that the large number from previous cycle is cropped and that max value is used instead. - assertEquals(MAX_SIZE, incompleteResultsThrottler.availableCapacity()); - } - - private long inversesU(int size, int sweetSpot) { - // Peak performance at sweetSPot. - int distance = Math.abs(sweetSpot - size); - return 1 + 20 * distance; - } - - /** - * A mock 'gateway' this is slower with more requests in-flight. It starts to become really much slower at - * 'breakPoint' number of parallel requests. - */ - class MockServer { - final LinkedList<Tuple2<Long, IncompleteResultsThrottler> > messageDoneByTime = new LinkedList<>(); - final int breakPoint; - final Random random = new Random(); - long time = 0; - - MockServer(int breakPoint) { - this.breakPoint = breakPoint; - } - - /** - * Figures out when next processed data will be ready. - * @return time in ms for next request to be finished. - */ - long nextRequestFinished() { - if (messageDoneByTime.isEmpty()) { - return Integer.MAX_VALUE; - } - return messageDoneByTime.peek().first; - } - - /** - * Advance simulation time and call finished on any requests. - * @param time to move to - */ - void moveTime(long time) { - this.time = time; - while (!messageDoneByTime.isEmpty() && messageDoneByTime.peek().first <= time) { - messageDoneByTime.pop().second.resultReady(true); - } - } - - /** - * New request. - * @param blocker do callback on blocker when request is done. - */ - void newRequest(IncompleteResultsThrottler blocker) { - long nextTime = (long)(20 + 0.1 * messageDoneByTime.size()); - - if (messageDoneByTime.size() > breakPoint) { - nextTime += (long) (40 + (random.nextDouble()) * 0.01 * messageDoneByTime.size()* messageDoneByTime.size()); - } - nextTime += time + random.nextInt()%4; - messageDoneByTime.push(new Tuple2<>(nextTime, blocker)); - } - } - - private static class Tuple2<T1, T2> { - - public final T1 first; - public final T2 second; - - public Tuple2(final T1 first, final T2 second) { - this.first = first; - this.second = second; - } - - @Override - public int hashCode() { throw new UnsupportedOperationException(); } - - @Override - public boolean equals(final Object obj) { throw new UnsupportedOperationException(); } - - @Override - public String toString() { - return "Tuple2(" + first + ", " + second + ")"; - } - - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java deleted file mode 100644 index 7799f6089db..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/core/operationProcessor/OperationProcessorTest.java +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.core.operationProcessor; - -import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.ConnectionParams; -import com.yahoo.vespa.http.client.config.Endpoint; -import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.Document; -import com.yahoo.vespa.http.client.core.EndpointResult; -import org.junit.Test; - -import java.time.Clock; -import java.util.ArrayDeque; -import java.util.Queue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * @author Einar M R Rosenvinge - */ -public class OperationProcessorTest { - - final Queue<Result> queue = new ArrayDeque<>(); - final Document doc1 = new Document("id:a:type::b", null, "data doc 1", null, Clock.systemUTC().instant()); - final Document doc1b = new Document("id:a:type::b", null, "data doc 1b", null, Clock.systemUTC().instant()); - final Document doc2 = new Document("id:a:type::b2", null, "data doc 2", null, Clock.systemUTC().instant()); - final Document doc3 = new Document("id:a:type::b3", null, "data doc 3", null, Clock.systemUTC().instant()); - - @Test - public void testBasic() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor q = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - - q.resultReceived(new EndpointResult("foo", new Result.Detail(null)), 0); - assertEquals(0, queue.size()); - - - q.sendDocument(doc1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("d"))), 3); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("e"))), 0); - assertEquals(1, queue.size()); - - //check a, b, c, d - Result aggregated = queue.poll(); - assertEquals("id:a:type::b", aggregated.getDocumentId()); - assertEquals(4, aggregated.getDetails().size()); - assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname()); - assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname()); - assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname()); - assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname()); - assertEquals("data doc 1", aggregated.getDocumentDataAsCharSequence().toString()); - - assertEquals(0, queue.size()); - - q.sendDocument(doc2); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("d"))), 3); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("e"))), 0); - assertEquals(1, queue.size()); - - // check a, b, c, d - aggregated = queue.poll(); - assertEquals("id:a:type::b2", aggregated.getDocumentId()); - assertEquals(4, aggregated.getDetails().size()); - assertEquals("a", aggregated.getDetails().get(0).getEndpoint().getHostname()); - assertEquals("b", aggregated.getDetails().get(1).getEndpoint().getHostname()); - assertEquals("c", aggregated.getDetails().get(2).getEndpoint().getHostname()); - assertEquals("d", aggregated.getDetails().get(3).getEndpoint().getHostname()); - assertEquals("data doc 2", aggregated.getDocumentDataAsCharSequence().toString()); - - assertEquals(0, queue.size()); - } - - @Test - public void testBlockingOfOperationsTwoEndpoints() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc1b); - - assertEquals(0, queue.size()); - // Only one operations should be in flight. - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(0, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertEquals(1, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(1, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertEquals(2, queue.size()); - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - // This should have no effect. - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 1); - assertEquals(2, queue.size()); - } - - @Test - public void testBlockingOfOperationsToSameDocIdWithTwoOperations() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc1b); - - assertEquals(0, queue.size()); - // Only one operations should be in flight. - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(1, queue.size()); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(2, queue.size()); - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - assertFalse(operationProcessor.oldestIncompleteResultId().isPresent()); - // This should have no effect. - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(2, queue.size()); - } - - @Test - public void testBlockingOfOperationsToSameDocIdMany() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - Queue<Document> documentQueue = new ArrayDeque<>(); - for (int x = 0; x < 100; x++) { - Document document = new Document("id:a:type::b", null, String.valueOf(x), null, Clock.systemUTC().instant()); - operationProcessor.sendDocument(document); - documentQueue.add(document); - } - - for (int x = 0; x < 100; x++) { - assertEquals(x, queue.size()); - // Only one operations should be in flight. - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - Document document = documentQueue.poll(); - operationProcessor.resultReceived(new EndpointResult(document.getOperationId(), new Result.Detail(Endpoint.create("host"))), 0); - assertEquals(x+1, queue.size()); - if (x < 99) { - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - } else { - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - } - } - } - - @Test - public void testMixOfBlockingAndNonBlocking() { - Endpoint endpoint = Endpoint.create("localhost"); - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(endpoint).build()) - .setConnectionParams(new ConnectionParams.Builder().build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc1b); // Blocked - operationProcessor.sendDocument(doc2); - operationProcessor.sendDocument(doc3); - - assertEquals(0, queue.size()); - assertEquals(3, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - // This should have no effect since it should not be sent. - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(3, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - - operationProcessor.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(2, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(1, operationProcessor.getIncompleteResultQueueSize()); - assertEquals(doc1b.getOperationId(), operationProcessor.oldestIncompleteResultId().get()); - operationProcessor.resultReceived(new EndpointResult(doc1b.getOperationId(), new Result.Detail(endpoint)), 0); - assertEquals(0, operationProcessor.getIncompleteResultQueueSize()); - assertFalse(operationProcessor.oldestIncompleteResultId().isPresent()); - } - - @Test - public void assertThatDuplicateResultsFromOneClusterWorks() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor q = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - q.sendDocument(doc1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 0); - assertEquals(0, queue.size()); - } - - @Test - public void testMultipleDuplicateDocIds() { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor q = new OperationProcessor( - new IncompleteResultsThrottler(1000, 1000, null, null), - (docId, documentResult) -> queue.add(documentResult), - sessionParams, null, Clock.systemUTC()); - - q.sendDocument(doc1); - assertEquals(0, queue.size()); - q.sendDocument(doc2); - assertEquals(0, queue.size()); - q.sendDocument(doc3); - - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(0, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("a"))), 0); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(1, queue.size()); - - q.resultReceived(new EndpointResult(doc2.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(2, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(2, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("c"))), 2); - assertEquals(2, queue.size()); - - q.resultReceived(new EndpointResult(doc3.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(3, queue.size()); - - q.resultReceived(new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("b"))), 1); - assertEquals(3, queue.size()); - assertEquals("data doc 1", queue.remove().getDocumentDataAsCharSequence().toString()); - assertEquals("data doc 2", queue.remove().getDocumentDataAsCharSequence().toString()); - assertEquals("data doc 3", queue.remove().getDocumentDataAsCharSequence().toString()); - } - - @Test - public void testWaitBlocks() throws InterruptedException { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(1, 1, null, null), - (docId, documentResult) -> {}, - sessionParams, null, Clock.systemUTC()); - - operationProcessor.sendDocument(doc1); - - final CountDownLatch started = new CountDownLatch(1); - final CountDownLatch done = new CountDownLatch(1); - - Thread shouldWait = new Thread(()-> { - started.countDown(); - operationProcessor.sendDocument(doc2); - done.countDown(); - }); - shouldWait.start(); - started.await(); - // We want the test to pass fast so we only wait 40mS to see that it is blocking. This might lead to - // some false positives, but that is ok. - assertFalse(done.await(40, TimeUnit.MILLISECONDS)); - operationProcessor.resultReceived( - new EndpointResult(doc1.getOperationId(), new Result.Detail(Endpoint.create("d"))), 0); - assertTrue(done.await(120, TimeUnit.SECONDS)); - - } - - @Test - public void testSendsResponseToQueuedDocumentOnClose() throws InterruptedException { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .build(); - - ScheduledThreadPoolExecutor executor = mock(ScheduledThreadPoolExecutor.class); - when(executor.awaitTermination(anyLong(), any())).thenReturn(true); - - CountDownLatch countDownLatch = new CountDownLatch(3); - - OperationProcessor operationProcessor = new OperationProcessor( - new IncompleteResultsThrottler(19, 19, null, null), - (docId, documentResult) -> { - countDownLatch.countDown(); - }, - sessionParams, executor, Clock.systemUTC()); - - // Will fail due to bogus host name, but will be retried. - operationProcessor.sendDocument(doc1); - operationProcessor.sendDocument(doc2); - operationProcessor.sendDocument(doc3); - - // Will create fail results. - operationProcessor.close(); - countDownLatch.await(); - } - - @Test - public void unknownHostThrowsExceptionAtConstructionTime() { - try { - SessionParams sessionParams = new SessionParams.Builder() - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("localhost")).build()) - .addCluster(new Cluster.Builder().addEndpoint(Endpoint.create("unknown.invalid")).build()) - .build(); - ScheduledThreadPoolExecutor executor = mock(ScheduledThreadPoolExecutor.class); - - CountDownLatch countDownLatch = new CountDownLatch(3); - - new OperationProcessor( - new IncompleteResultsThrottler(19, 19, null, null), - (docId, documentResult) -> { - countDownLatch.countDown(); - }, - sessionParams, executor, Clock.systemUTC()); - - fail("Expected exception"); - } - catch (IllegalArgumentException e) { - assertEquals("Unknown host: unknown.invalid:4080 ssl=false", e.getMessage()); - } - } - -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java deleted file mode 100644 index bf4d4d0800b..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/handlers/V3MockParsingRequestHandler.java +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.handlers; - -import com.yahoo.vespa.http.client.core.Encoder; -import com.yahoo.vespa.http.client.core.ErrorCode; -import com.yahoo.vespa.http.client.core.Headers; -import com.yahoo.vespa.http.client.core.OperationStatus; -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.handler.AbstractHandler; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.27 - */ -public class V3MockParsingRequestHandler extends AbstractHandler { - private final int responseCode; - private volatile Scenario scenario; - private final BlockingQueue<CountDownLatch> delayedRequests = new LinkedBlockingQueue<>(); - private final AtomicBoolean delayedResponseShouldBlock = new AtomicBoolean(true); - public final AtomicBoolean badRequestScenarioShouldReturnBadRequest = new AtomicBoolean(false); - private final String name; - private static final AtomicInteger sessionIdGenerator = new AtomicInteger(0); - private AtomicInteger internalCounter = new AtomicInteger(0); - - public enum Scenario { - ALL_OK, RETURN_WRONG_SESSION_ID, - DISCONNECT_IMMEDIATELY, DONT_ACCEPT_VERSION, RETURN_UNEXPECTED_VERSION, - INTERNAL_SERVER_ERROR, COULD_NOT_FEED, MBUS_RETURNED_ERROR, - NEVER_RETURN_ANY_RESULTS, DELAYED_RESPONSE, BAD_REQUEST, SERVER_ERROR_TWICE_THEN_OK, - EXPECT_HIGHEST_PRIORITY_AND_TRACELEVEL_123, CONDITON_NOT_MET - } - - public V3MockParsingRequestHandler() { - this("", HttpServletResponse.SC_OK, Scenario.ALL_OK); - } - - public V3MockParsingRequestHandler(String name) { - this(name, HttpServletResponse.SC_OK, Scenario.ALL_OK); - } - - public V3MockParsingRequestHandler(int responseCode) { - this("", responseCode, Scenario.ALL_OK); - } - - public V3MockParsingRequestHandler(int responseCode, Scenario scenario) { - this("", responseCode, scenario); - } - - public V3MockParsingRequestHandler(String name, int responseCode, Scenario scenario) { - this.name = name; - this.responseCode = responseCode; - this.scenario = scenario; - } - - public void setScenario(Scenario scenario) { - this.scenario = scenario; - } - - @Override - public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - System.err.println("Server " + name + " got request from: " + request.getHeader(Headers.SESSION_ID)); - switch (scenario) { - case ALL_OK: - allOk(baseRequest, request, response); - break; - case RETURN_WRONG_SESSION_ID: - wrongSessionId(baseRequest, request, response); - break; - case DISCONNECT_IMMEDIATELY: - disconnect(baseRequest, response); - break; - case DONT_ACCEPT_VERSION: - dontAcceptVersion(baseRequest, request, response); - break; - case RETURN_UNEXPECTED_VERSION: - unexpectedVersion(baseRequest, request, response); - break; - case INTERNAL_SERVER_ERROR: - internalServerError(baseRequest, request, response); - break; - case COULD_NOT_FEED: - couldNotFeed(baseRequest, request, response); - break; - case MBUS_RETURNED_ERROR: - mbusReturnedError(baseRequest, request, response); - break; - case NEVER_RETURN_ANY_RESULTS: - neverReturnAnyResults(baseRequest, request, response); - break; - case DELAYED_RESPONSE: - delayedResponse(baseRequest, request, response); - break; - case BAD_REQUEST: - badRequest(baseRequest, request, response); - break; - case SERVER_ERROR_TWICE_THEN_OK: - int state = internalCounter.getAndIncrement(); - if (state >= 2) { - allOk(baseRequest, request, response); - } else { - couldNotFeed(baseRequest, request, response); - } - break; - case EXPECT_HIGHEST_PRIORITY_AND_TRACELEVEL_123: - checkIfSessionThenHighPriorityAndTraceLevel123(request); - allOk(baseRequest, request, response); - break; - case CONDITON_NOT_MET: - conditionNotMetRequest(baseRequest, request, response); - break; - default: - throw new IllegalArgumentException("Test scenario " + scenario + " not supported."); - } - } - - private void checkIfSessionThenHighPriorityAndTraceLevel123(HttpServletRequest request) { - if (request.getHeader(Headers.SESSION_ID) != null) { - assert (request.getHeader(Headers.PRIORITY).equals("HIGHEST")); - assert (request.getHeader(Headers.TRACE_LEVEL).equals("123")); - } - } - - private void conditionNotMetRequest(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondConditionNotMet(responseWriter, operationId); - } - closeChannel(responseWriter); - - } - private void badRequest(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - if (badRequestScenarioShouldReturnBadRequest.get()) { - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - BufferedReader reader = new BufferedReader(new InputStreamReader(request.getInputStream())); - while (reader.readLine() != null) { - //consume input, not really needed? - } - reader.close(); - closeChannel(responseWriter); - } else { - allOk(baseRequest, request, response); - } - } - - private void delayedResponse(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - if (delayedResponseShouldBlock.get()) { - CountDownLatch latch = new CountDownLatch(1); - delayedRequests.add(latch); - try { - latch.await(120, TimeUnit.SECONDS); //wait "forever" - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - if (latch.getCount() != 0L) { - throw new RuntimeException("Delayed request handler did not get poke()d."); - } - } else { - } - allOk(baseRequest, request, response); - } - - public void poke() throws InterruptedException { - CountDownLatch latch = delayedRequests.poll(10, TimeUnit.SECONDS); - latch.countDown(); - } - - public void pokeAllAndUnblockFromNowOn() { - delayedResponseShouldBlock.set(false); - while (!delayedRequests.isEmpty()) { - CountDownLatch latch = delayedRequests.remove(); - latch.countDown(); - } - } - - private void allOk(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondOK(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void wrongSessionId(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = generateMockSessionId(); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondOK(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void disconnect(Request baseRequest, HttpServletResponse response) throws IOException { - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - closeChannel(responseWriter); - } - - private void dontAcceptVersion(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(Headers.HTTP_NOT_ACCEPTABLE); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - responseWriter.write("Go away, no such version."); - responseWriter.flush(); - closeChannel(responseWriter); - } - - private void unexpectedVersion(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - response.setHeader(Headers.SESSION_ID, sessionId); - response.setHeader(Headers.VERSION, "12345678"); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondOK(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void internalServerError(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(500); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - responseWriter.write("boom"); - responseWriter.flush(); - closeChannel(responseWriter); - } - - private void couldNotFeed(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondTransientFailed(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void mbusReturnedError(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - String operationId; - while ((operationId = readOperationId(request.getInputStream())) != null) { - long lengthToSkip = readByteLength(request.getInputStream()); - while (lengthToSkip > 0) { - long skipped = request.getInputStream().skip(lengthToSkip); - lengthToSkip -= skipped; - } - respondFailedWithTransitiveErrorSeenFromClient(responseWriter, operationId); - } - closeChannel(responseWriter); - } - - private void neverReturnAnyResults(Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException { - String sessionId = getSessionId(request); - setHeaders(response, sessionId); - response.setStatus(responseCode); - baseRequest.setHandled(true); - PrintWriter responseWriter = response.getWriter(); - BufferedReader reader = new BufferedReader(new InputStreamReader(request.getInputStream())); - while (reader.readLine() != null) { - //consume input, not really needed? - } - reader.close(); - closeChannel(responseWriter); - } - - void closeChannel(PrintWriter responseWriter) { - System.err.println("Mock server " + name + " closing channel."); - responseWriter.close(); - } - - private String readOperationId(InputStream requestInputStream) throws IOException { - StringBuilder idBuf = new StringBuilder(100); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 32) { - break; - } - idBuf.append((char) c); //it's ASCII - } - if (c == -1) { - return null; - } - return Encoder.decode(idBuf.toString(), new StringBuilder(idBuf.length())).toString(); - } - - private int readByteLength(InputStream requestInputStream) throws IOException { - StringBuilder lenBuf = new StringBuilder(8); - int c; - while ((c = requestInputStream.read()) != -1) { - if (c == 10) { - break; - } - lenBuf.append((char) c); //it's ASCII - } - if (lenBuf.length() == 0) { - throw new IllegalStateException("Operation length missing."); - } - return Integer.valueOf(lenBuf.toString(), 16); - } - - private static void setHeaders(HttpServletResponse response, String sessionId) { - response.setHeader(Headers.SESSION_ID, sessionId); - response.setHeader(Headers.VERSION, "3"); - } - - private void respondFailed(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = - new OperationStatus("mbus returned boom", docId, ErrorCode.ERROR, false, "trace"); - writeResponse(responseWriter, operationStatus); - } - - private void respondTransientFailed(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = new OperationStatus( - "Could not put", docId, ErrorCode.TRANSIENT_ERROR, false, ""); - writeResponse(responseWriter, operationStatus); - } - - private void respondFailedWithTransitiveErrorSeenFromClient(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = - new OperationStatus("NETWORK_ERROR", docId, ErrorCode.ERROR, false, "trace"); - writeResponse(responseWriter, operationStatus); - } - - private void respondConditionNotMet(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = - new OperationStatus("this is a test", docId, ErrorCode.ERROR, true, "trace"); - writeResponse(responseWriter, operationStatus); - } - private void respondOK(PrintWriter responseWriter, String docId) { - final OperationStatus operationStatus = new OperationStatus("Doc fed", docId, ErrorCode.OK, false, "Trace message"); - writeResponse(responseWriter, operationStatus); - } - - private void writeResponse(PrintWriter responseWriter, - final OperationStatus operationStatus) { - responseWriter.print(operationStatus.render()); - responseWriter.flush(); - System.err.println("Mock " + name + " server wrote: " + operationStatus.render()); - } - - private String getSessionId(HttpServletRequest request) { - return request.getHeader(Headers.CLIENT_ID); - } - - private String generateMockSessionId() { - return String.valueOf(sessionIdGenerator.getAndIncrement()); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java deleted file mode 100644 index 55f6af9ed32..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.yahoo.vespa.http.client.config.Cluster; -import com.yahoo.vespa.http.client.config.FeedParams; -import com.yahoo.vespa.http.client.config.SessionParams; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.assertThat; - -public class CommandLineArgumentsTest { - - private String[] asArray() { - String[] array = new String[args.size()]; - args.toArray(array); - return array; - } - - private void add(String key, String value) { - args.add("--" + key); - args.add(value); - } - - private void addMinimum() { - add("host", "hostValue"); - } - - private ArrayList<String> args = new ArrayList<>(); - - @Test - public void testRequiredFlags() { - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - add("file", "fileValue"); - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - args.clear(); - addMinimum(); - assertThat(CommandLineArguments.build(asArray()), is(not(nullValue()))); - } - - @Test - public void testStreaming() { - add("host", "hostValue"); - add("file", null); // Not yet implemented support for streaming - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - } - - @Test - public void testBrokenFlags() { - addMinimum(); - args.add("FOO"); - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - } - - @Test - public void testBadPriority() { - addMinimum(); - add("priority", "non existing"); - assertThat(CommandLineArguments.build(asArray()), is(nullValue())); - } - - @Test - public void testOkPriority() { - addMinimum(); - add("priority", "HIGHEST"); - assertThat(CommandLineArguments.build(asArray()).createSessionParams(false).getFeedParams().getPriority(), - is("HIGHEST")); - } - - @Test - public void testDefaults() { - addMinimum(); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(false /* use json */); - assertThat(params.getClientQueueSize(), is(10000)); - assertThat(params.getThrottlerMinSize(), is(0)); - assertThat(params.getClusters().size(), is(1)); - assertThat(params.getClusters().get(0).getEndpoints().size(), is(1)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getHostname(), is("hostValue")); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(4080)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); - assertThat(params.getConnectionParams().getUseCompression(), is(false)); - assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(4)); - assertThat(params.getFeedParams().getRoute(), is("default")); - assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.XML_UTF8)); - assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(180000L)); - assertThat(params.getFeedParams().getMaxInFlightRequests(), is(10000)); - assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(180000L)); - } - - @Test - public void testAllImplementedFlags() { - add("file", "fileValue.json"); - add("route", "routeValue"); - add("host", "hostValue"); - add("port", "1234"); - add("timeout", "2345"); - add("numPersistentConnectionsPerEndpoint", "7"); - args.add("--useCompression"); - args.add("--useDynamicThrottling"); - add("maxpending", "3456"); - args.add("--verbose"); - args.add("--useTls"); - add("header", "Header-Name: Header-Value"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - assertThat(params.getClientQueueSize(), is(3456)); - assertThat(params.getThrottlerMinSize(), is(10)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(1234)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(true)); - assertThat(params.getConnectionParams().getUseCompression(), is(true)); - assertThat(params.getConnectionParams().getHeaders().size(), is(1)); - assertThat(params.getFeedParams().getRoute(), is("routeValue")); - assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.JSON_UTF8)); - assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(2345000L)); - assertThat(params.getFeedParams().getMaxInFlightRequests(), is(3456)); - assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(2345000L)); - assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(7)); - } - - @Test - public void testAddingMultipleHttpHeaders() { - add("host", "hostValue"); - String header1Name = "Header-Name-1"; - String header1Value = "Header-Value"; - add("header", header1Name + ": " + header1Value); - String header2Name = "Header-Name-2"; - String header2Value = "Another-Header-Value"; - add("header", header2Name + ": " + header2Value); - - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - - List<Map.Entry<String, String>> headers = new ArrayList<>(params.getConnectionParams().getHeaders()); - headers.sort(Comparator.comparing(Map.Entry::getKey)); - - assertThat(headers.size(), is(2)); - Map.Entry<String, String> actualHeader1 = headers.get(0); - assertThat(actualHeader1.getKey(), is(header1Name)); - assertThat(actualHeader1.getValue(), is(header1Value)); - Map.Entry<String, String> actualHeader2 = headers.get(1); - assertThat(actualHeader2.getKey(), is(header2Name)); - assertThat(actualHeader2.getValue(), is(header2Value)); - } - - @Test - public void testMultiHost() { - add("file", "fileValue.json"); - add("port", "1234"); - add("host", "hostValue1,hostValue2, hostValue3"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - assertThat(params.getClusters().size(), is(3)); - final Set<String> hosts = new HashSet<>(); - for (Cluster cluster : params.getClusters()) { - assertThat(cluster.getEndpoints().size(), is(1)); - hosts.add(cluster.getEndpoints().get(0).getHostname()); - assertThat(cluster.getEndpoints().get(0).getPort(), is(1234)); - } - assertThat(hosts, hasItem("hostValue1")); - assertThat(hosts, hasItem("hostValue2")); - assertThat(hosts, hasItem("hostValue3")); - } - - @Test - public void testUseV3Protocol() { - addMinimum(); - args.add("--useV3Protocol"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true /* use json */); - } - - @Test - public void testEndpoint() { - add("endpoint", "http://myendpoint:1234"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getHostname(), is("myendpoint")); - assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(1234)); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); - } - - @Test - public void testEndpointHttps() { - add("endpoint", "https://myendpoint:1234"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - SessionParams params = arguments.createSessionParams(true); - assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(true)); - } - - @Test - public void testEndpointAndHost() { - add("host", "myhost"); - add("port", "2345"); - add("endpoint", "http://myendpoint:1234"); - CommandLineArguments arguments = CommandLineArguments.build(asArray()); - assertThat(arguments, is(nullValue())); // cannot have both endpoint and host - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java deleted file mode 100644 index 9c77ed9a6d9..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/FormatInputStreamTest.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.util.Optional; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -/** - * @author valerijf - */ -public class FormatInputStreamTest { - @Test(expected=IllegalArgumentException.class) - public void testWithGarbageText() throws IOException { - String streamString = "This is neither XML nor JSON!"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - } - - @Test - public void testWithFileInput() throws IOException { - String fileString = "{\"format\": \"json\"}"; - File file = File.createTempFile("feeddata", "json"); - file.deleteOnExit(); - try (FileWriter writer = new FileWriter(file)) { - writer.write(fileString); - } - - FormatInputStream formatInputStream = new FormatInputStream(null, Optional.of(file.getAbsolutePath()), false); - assertThat(fileString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON)); - } - - @Test - public void testPreferenceFileOverStream() throws IOException { - String streamString = "something entirely different"; - String fileString = "{\"format\": \"json\"}"; - - InputStream jsonStream = getInputStreamOf(streamString); - File file = File.createTempFile("feeddata", "json"); - file.deleteOnExit(); - try (FileWriter writer = new FileWriter(file)) { - writer.write(fileString); - } - - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.of(file.getAbsolutePath()), false); - assertThat(fileString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON)); - } - - @Test - public void testSimpleJsonInputStream() throws IOException { - String streamString = "{\"format\": \"json\"}"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - - assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.JSON)); - } - - @Test - public void testSimpleXmlInputStream() throws IOException { - String streamString = "<scope><tag>format</tag><value>xml</value></scope>"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - - assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML)); - } - - @Test - public void testSparselyFormattedXml() throws IOException { - String streamString = " \t\t\n<scope>\n\n\n<tag>format</tag><value>xml</value></scope>"; - InputStream jsonStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(jsonStream, Optional.empty(), false); - - assertThat(streamString, is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML)); - } - - @Test - public void testAddRootToXml() throws IOException { - String streamString = "some random text"; - InputStream textStream = getInputStreamOf(streamString); - FormatInputStream formatInputStream = new FormatInputStream(textStream, Optional.empty(), true); - - assertThat("<vespafeed>" + streamString + "</vespafeed>", - is(convertStreamToString(formatInputStream.getInputStream()))); - assertThat(formatInputStream.getFormat(), is(FormatInputStream.Format.XML)); - } - - private static String convertStreamToString(InputStream inputStream) throws IOException { - StringBuilder builder = new StringBuilder(); - while (true) { - int character = inputStream.read(); - if (character == -1) { - inputStream.close(); - return builder.toString(); - } - builder.append((char)character); - } - } - - private static InputStream getInputStreamOf(String text) { - return new ByteArrayInputStream(text.getBytes()); - } -} diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java deleted file mode 100644 index 0a5b3771958..00000000000 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/JsonReaderTest.java +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.client.runner; - -import com.yahoo.vespa.http.client.FeedClient; -import com.yahoo.vespa.http.client.core.JsonReader; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.yahoo.test.json.JsonTestHelper.inputJson; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; - -public class JsonReaderTest { - - private static String doc1_id = "id:unittest:testMapStringToArrayOfInt::whee"; - - private static String doc1 = inputJson( - "{", - " 'update': '"+ doc1_id + "',", - " 'fields': {", - " 'actualMapStringToArrayOfInt': {", - " 'assign': [", - " { 'key': 'bamse', 'value': [ 2, 1, 3] }", - " ]", - " }", - " }", - "}"); - - private static String doc2_id = "id:unittest:smoke::whee"; - - private static String doc2 = inputJson( - "{", - " 'put': '" + doc2_id + "',", - " 'fields': {", - " 'something': 'smoketest',", - " 'nalle': 'bamse'", - " }", - "}"); - - private static String doc3 = inputJson( - "{", - " 'update': 'id:unittest:testarray::whee',", - " 'fields': {", - " 'actualarray': {", - " 'add': [", - " 'person naïve',", - " 'another person'", - " ]", - " }", - " }", - "}"); - - private static String doc4 = inputJson( - "{", - " 'remove': '" + doc2_id + "'", - "}"); - - private static String doc5_id = "id:unittest:smoking::wheels"; - - private static String doc5 = inputJson( - "{", - " 'id': '" + doc5_id + "',", - " 'fields': {", - " 'something': 'smoketest',", - " 'nalle': 'bamse'", - " }", - "}"); - - private static class TestFeedClient implements FeedClient { - - public List<String> documentIds = new ArrayList<>(); - public List<CharSequence> datas = new ArrayList<>(); - public List<Object> contexts = new ArrayList<>(); - - @Override - public void stream(String documentId, CharSequence documentData) { - stream(documentId, documentData, null); - } - - @Override - public void stream(String documentId, String operationId, CharSequence documentData, Object context) { - documentIds.add(documentId); - datas.add(documentData); - contexts.add(context); - } - - @Override - public void close() { } - - @Override - public String getStatsAsJson() { return null; } - } - - final TestFeedClient session = new TestFeedClient(); - final AtomicInteger numSent = new AtomicInteger(0); - - @Test - public void testReadNoDocument() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - " ".getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(0)); - } - - @Test - public void testReadOneDocument() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - ("["+ doc1 + "]" ).getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(1)); - assertThat(session.documentIds.get(0), is(doc1_id)); - assertThat(session.datas.size(), is(1)); - assertThat(session.datas.get(0), is(doc1)); - } - - @Test - public void testReadFourDocuments() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - (" [ "+ doc1 + " , " + doc2 + ", " + doc3 + "," + doc4 + " ] ").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(4)); - assertThat(session.documentIds.get(0), is(doc1_id)); - assertThat(session.documentIds.get(1), is(doc2_id)); - assertThat(session.datas.size(), is(4)); - assertThat(session.datas.get(0), is(doc1)); - assertThat(session.datas.get(1).toString(), is(doc2)); - assertThat(session.datas.get(2).toString(), is(doc3)); - assertThat(session.datas.get(3).toString(), is(doc4)); - } - - @Test - public void testDocWithIdAndNotPut() throws Exception { - InputStream inputStream = new ByteArrayInputStream( - (" [ "+ doc5 + " ] ").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(1)); - assertThat(session.documentIds.get(0), is(doc5_id)); - } - - @Test - public void simpleMicroBenchmarkTest() throws Exception { - StringBuilder stream = new StringBuilder(); - stream.append("["); - int docsInStream = 15000; - for (int x = 0; x < docsInStream -1; x++) { - if (x % 10 == 0) { - stream.append(doc1 + ", "); - } else { - // Add some randomness to the layout to trigger potential bugs in parsing. - stream.append("{\"remove\": \"id:unittest:smoke::whee"); - for (int y = 0 ; y < x % 277 ; y++) { - stream.append("X"); - } - stream.append("\"}, "); - } - } - stream.append(doc3); - stream.append("]"); - - InputStream inputStream = new ByteArrayInputStream(stream.toString().getBytes(StandardCharsets.UTF_8)); - long startTime = System.currentTimeMillis(); - JsonReader.read(inputStream, session, numSent); - // At time of writing, it took about 200 ms on my mac. - System.err.println("Run time is " + (System.currentTimeMillis() - startTime) + " ms"); - inputStream.close(); - - // Verify that content is not rubbish. - for (int x = 0; x < docsInStream - 1; x++) { - if (x % 10 == 0) { - assertThat(session.datas.get(x).toString(), is(doc1)); - assertThat(session.documentIds.get(x), is(doc1_id)); - } - } - assertThat(session.datas.get(docsInStream-1).toString(), is(doc3)); - assertThat(numSent.get(), is(docsInStream)); - } - - @Test(expected=RuntimeException.class) - public void testBadJsonCommaAfterLastElement() { - InputStream inputStream = new ByteArrayInputStream( - ("["+ doc1 + ",]" ).getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - } - - @Test(expected=RuntimeException.class) - public void testTotalGarbage() { - InputStream inputStream = new ByteArrayInputStream(("garbage" ).getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - } - - @Test(expected=RuntimeException.class) - public void testTwoDocIds() { - InputStream inputStream = new ByteArrayInputStream(("[{\"remove\": \"id\", \"update\": \"id:\"}]" - .getBytes(StandardCharsets.UTF_8))); - JsonReader.read(inputStream, session, numSent); - } - - @Test(expected = IllegalArgumentException.class) - public void throwsOnMissingId() { - InputStream inputStream = new ByteArrayInputStream( - inputJson("[{'fields':{ 'something': 'smoketest', 'nalle': 'bamse' }}]").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - } - - @Test - public void testFullDocument() throws Exception { - InputStream inputStream = new ByteArrayInputStream(( - "[{\n" + - " \"update\": \"id:foo:music:doc:foo:bar\",\n" + - " \n" + - " \"fields\": {\n" + - " \"artist\": {\n" + - " \"assign\": null" + - " },\n" + - " \n" + - " \"albums\": {\n" + - " \"assign\": [\n" + - " \"Kramgoda laatar 4\",\n" + - " \"Kramgoda laatar 5\",\n" + - " \"Kramgoda laatar 6\"\n" + - " ],\n" + - " \"add\": [\n" + - " \"Kramgoda laatar 7\",\n" + - " \"Kramgoda laatar 8\"\n" + - " ]\n" + - " },\n" + - " \"inceptionYear\": {\n" + - " \"increment\": 4\n" + - " },\n" + - " \"concerts\": {\n" + - " \"assign\": {\n" + - " \"Torsby 1993\": 1000,\n" + - " \"Uddevalla 2000\": 34\n" + - " },\n" + - " \"match\": {\n" + - " \"element\": \"Sundsvall 1980\",\n" + - " \"increment\": 5392\n" + - " },\n" + - " \"add\": {\n" + - " \"Kiruna 1999\": 200,\n" + - " \"Oslo 1998\": 2000\n" + - " }\n" + - " },\n" + - " \"scores\": {\n" + - " \"match\": {\n" + - " \"element\": \"Sven Ingvars\",\n" + - " \"match\": {\n" + - " \"element\": 0,\n" + - " \"increment\": 78\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}]\n").getBytes(StandardCharsets.UTF_8)); - JsonReader.read(inputStream, session, numSent); - inputStream.close(); - assertThat(session.documentIds.size(), is(1)); - assertThat(session.documentIds.get(0), is("id:foo:music:doc:foo:bar")); - } -} |