From c9770112336d353dc5e69357ffbd0f830d08068e Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 23 Jun 2021 19:07:46 +0200 Subject: Support concatenated JSON objects as feed format --- .../main/java/ai/vespa/feed/client/JsonFeeder.java | 12 +++--- .../java/ai/vespa/feed/client/JsonFeederTest.java | 44 ++++++++++++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) (limited to 'vespa-feed-client') diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonFeeder.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonFeeder.java index 0ba373eef18..aa4cd40f952 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonFeeder.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/JsonFeeder.java @@ -359,16 +359,14 @@ public class JsonFeeder implements Closeable { abstract String getDocumentJson(long start, long end); CompletableFuture next() throws IOException { - if (multipleOperations && !arrayPrefixParsed){ - expect(START_ARRAY); + JsonToken token = parser.nextToken(); + if (multipleOperations && ! arrayPrefixParsed && token == START_ARRAY) { arrayPrefixParsed = true; + token = parser.nextToken(); } - - JsonToken token = parser.nextToken(); if (token == END_ARRAY && multipleOperations) return null; - else if (token == null && !multipleOperations) return null; - else if (token == START_OBJECT); - else throw new OperationParseException("Unexpected token '" + parser.currentToken() + "' at offset " + parser.getTokenLocation().getByteOffset()); + else if (token == null && ! arrayPrefixParsed) return null; + else if (token != START_OBJECT) throw new OperationParseException("Unexpected token '" + parser.currentToken() + "' at offset " + parser.getTokenLocation().getByteOffset()); long start = 0, end = -1; OperationType type = null; DocumentId id = null; diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/JsonFeederTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/JsonFeederTest.java index 3e0f886a40a..6c26e1d8ae8 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/JsonFeederTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/JsonFeederTest.java @@ -3,11 +3,13 @@ package ai.vespa.feed.client; import org.junit.jupiter.api.Test; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -76,6 +78,48 @@ class JsonFeederTest { } } + @Test + public void multipleJsonArrayOperationsAreDispatchedToFeedClient() throws IOException, ExecutionException, InterruptedException { + SimpleClient client = new SimpleClient(); + try (JsonFeeder feeder = JsonFeeder.builder(client).build()) { + String json = "[{" + + " \"put\": \"id:ns:type::abc1\",\n" + + " \"fields\": {\n" + + " \"lul\":\"lal\"\n" + + " }\n" + + "},\n" + + "{" + + " \"put\": \"id:ns:type::abc2\",\n" + + " \"fields\": {\n" + + " \"lul\":\"lal\"\n" + + " }\n" + + "}]\n"; + feeder.feedMany(new ByteArrayInputStream(json.getBytes(UTF_8))).get(); + assertEquals(new HashSet<>(Arrays.asList("abc1", "abc2")), client.ids); + } + } + + @Test + public void multipleJsonLOperationsAreDispatchedToFeedClient() throws IOException, ExecutionException, InterruptedException { + SimpleClient client = new SimpleClient(); + try (JsonFeeder feeder = JsonFeeder.builder(client).build()) { + String json = "{" + + " \"put\": \"id:ns:type::abc1\",\n" + + " \"fields\": {\n" + + " \"lul\":\"lal\"\n" + + " }\n" + + "}\n" + + "{" + + " \"put\": \"id:ns:type::abc2\",\n" + + " \"fields\": {\n" + + " \"lul\":\"lal\"\n" + + " }\n" + + "}\n"; + feeder.feedMany(new ByteArrayInputStream(json.getBytes(UTF_8))).get(); + assertEquals(new HashSet<>(Arrays.asList("abc1", "abc2")), client.ids); + } + } + @Test public void singleJsonOperationIsDispatchedToFeedClient() throws IOException, ExecutionException, InterruptedException { try (JsonFeeder feeder = JsonFeeder.builder(new SimpleClient()).build()) { -- cgit v1.2.3