Add streamed fill test skeletonbratseth/streamed-fill

7 files changed, 82 insertions, 27 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java
index 5d26a71137e..8c6f3df7ca1 100644
--- a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java
@@ -21,7 +21,7 @@ public class HitField {
     private final String rawContent;
     private final boolean isCJK;
 
-    private boolean xmlProperty;
+    private final boolean xmlProperty;
 
     private List<FieldPart> tokenizedContent = null;
     private String content;
@@ -191,8 +191,8 @@ public class HitField {
     private List<FieldPart> tokenizePretokenized() {
         String[] pre = rawContent.split("\u001F+");
         List<FieldPart> tokenized = new ArrayList<>(pre.length);
-        for (int i = 0; i < pre.length; i++) {
-            tokenized.add(createToken(pre[i], true));
+        for (String s : pre) {
+            tokenized.add(createToken(s, true));
         }
         return tokenized;
     }
@@ -245,9 +245,7 @@ public class HitField {
      */
     public void setTokenizedContent(List<FieldPart> list) {
         tokenizedContent = new ArrayList<>(list.size());
-        for (Iterator<FieldPart> i = list.iterator(); i.hasNext(); ) {
-            tokenizedContent.add(i.next());
-        }
+        tokenizedContent.addAll(list);
         // Must null content reference _before_ calling getContent()
         content = null;
     }
@@ -262,9 +260,8 @@ public class HitField {
     public String getContent() {
         if (content == null) {
             StringBuilder buf = new StringBuilder();
-            Iterator<FieldPart> iter = ensureTokenized().iterator();
-            while(iter.hasNext()) {
-                buf.append(iter.next().getContent());
+            for (FieldPart fieldPart : ensureTokenized()) {
+                buf.append(fieldPart.getContent());
             }
             content = buf.toString();
         }
@@ -274,9 +271,7 @@ public class HitField {
     /** Returns the content of this field, using the arguments as bolding tags */
     public String getContent(String boldOpenTag, String boldCloseTag, String separatorTag) {
         StringBuilder b = new StringBuilder();
-        Iterator<FieldPart> iter = ensureTokenized().iterator();
-        while(iter.hasNext()) {
-            FieldPart f = iter.next();
+        for (FieldPart f : ensureTokenized()) {
             if (f instanceof BoldOpenFieldPart && boldOpenTag != null && boldOpenTag.length() > 0)
                 b.append(boldOpenTag);
             else if (f instanceof BoldCloseFieldPart && boldCloseTag != null && boldCloseTag.length() > 0)
@@ -316,9 +311,7 @@ public class HitField {
                                 String separatorTag,
                                 boolean inAttribute) {
         StringBuilder xml = new StringBuilder();
-        Iterator<FieldPart> iter = ensureTokenized().iterator();
-        while(iter.hasNext()) {
-            FieldPart f = iter.next();
+        for (FieldPart f : ensureTokenized()) {
             if (f instanceof BoldOpenFieldPart
                 && boldOpenTag != null
                 && boldOpenTag.length() > 0)
@@ -344,9 +337,7 @@ public class HitField {
      */
     public String bareContent(boolean XMLQuote, boolean inAttribute) {
         StringBuilder bareContent = new StringBuilder();
-        Iterator<FieldPart> iter = ensureTokenized().iterator();
-        while(iter.hasNext()) {
-            FieldPart f = iter.next();
+        for (FieldPart f : ensureTokenized()) {
             if (f instanceof MarkupFieldPart)
                 continue;
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java
index 09dafe880f0..b55a4ff9b71 100644
--- a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java
+++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java
@@ -44,10 +44,10 @@ public class XmlRenderer {
                 byte[] data = value.asData();
                 renderTarget.append("<data length=\"").append(data.length);
                 renderTarget.append("\" encoding=\"hex\">");
-                for (int i = 0; i < data.length; i++) {
+                for (byte datum : data) {
                     for (int sh = 4; sh >= 0; sh -= 4) {
-                        int val = (data[i] >> sh) & 0xF;
-                        char hexdigit = (val < 10) ? ((char)('0' + val)) : ((char)('A' + val - 10));
+                        int val = (datum >> sh) & 0xF;
+                        char hexdigit = (val < 10) ? ((char) ('0' + val)) : ((char) ('A' + val - 10));
                         renderTarget.append(hexdigit);
                     }
                 }
@@ -84,9 +84,7 @@ public class XmlRenderer {
         }
 
         private void indent(int nestingLevel) {
-            for (int i = 0; i < nestingLevel; ++i) {
-                renderTarget.append("  ");
-            }
+            renderTarget.append("  ".repeat(Math.max(0, nestingLevel)));
         }
 
         private void renderMap(Inspector sequence, int nestingLevel) {
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
index 44620179c1d..05df0883f07 100644
--- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
+++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
@@ -422,7 +422,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
     protected void renderGroupMetadata(GroupId id) throws IOException {
         if (!(id instanceof ValueGroupId || id instanceof BucketGroupId)) return;
 
-        if (id instanceof ValueGroupId valueId) {
+        if (id instanceof ValueGroupId<?> valueId) {
             generator.writeStringField(GROUPING_VALUE, getIdValue(valueId));
         } else {
             BucketGroupId<?> bucketId = (BucketGroupId<?>) id;
diff --git a/container-search/src/test/java/com/yahoo/search/handler/StreamedFillTest.java b/container-search/src/test/java/com/yahoo/search/handler/StreamedFillTest.java
new file mode 100644
index 00000000000..bea16fbbdbc
--- /dev/null
+++ b/container-search/src/test/java/com/yahoo/search/handler/StreamedFillTest.java
@@ -0,0 +1,60 @@
+package com.yahoo.search.handler;
+
+import ai.vespa.cloud.ZoneInfo;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.container.core.ContainerHttpConfig;
+import com.yahoo.container.handler.threadpool.ContainerThreadPool;
+import com.yahoo.container.jdisc.ContentChannelOutputStream;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.jdisc.handler.ReadableContentChannel;
+import com.yahoo.jdisc.test.MockMetric;
+import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry;
+import com.yahoo.search.searchchain.ExecutionFactory;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Tests filling hits as they are rendered to avoid having to keep all hits in memory.
+ *
+ * @author bratseth
+ */
+public class StreamedFillTest {
+
+    @Test
+    void testStreamedFill() {
+        var handler = new SearchHandler(new MockMetric(),
+                                        new SimpleContainerThreadpool(),
+                                        new CompiledQueryProfileRegistry(),
+                                        new ContainerHttpConfig.Builder().build(),
+                                        new ComponentRegistry(),
+                                        ExecutionFactory.empty(),
+                                        ZoneInfo.defaultInfo());
+        var request = HttpRequest.createTestRequest("?query=foo", com.yahoo.jdisc.http.HttpRequest.Method.GET);
+        var response = (HttpSearchResponse)handler.handle(request);
+        System.out.println(render(response));
+    }
+
+    private String render(HttpSearchResponse response) {
+        try {
+            var out = new ReadableContentChannel();
+            response.render(new ContentChannelOutputStream(out), out, null);
+            return new String(out.toStream().readAllBytes(), UTF_8);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static class SimpleContainerThreadpool implements ContainerThreadPool {
+
+        private final Executor executor = Executors.newCachedThreadPool();
+
+        @Override public Executor executor() { return executor; }
+
+    }
+
+}
diff --git a/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java
index c038b3c67c9..f51b2c2a216 100644
--- a/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java
@@ -31,7 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 /**
  * Test adding hits to a hit group during rendering.
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public class AsyncGroupPopulationTestCase {
     private static class WrappedFuture<F> extends CompletableFuture<F> {
diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
index 6db3ec9735e..9cc2b4e6227 100644
--- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
@@ -1482,7 +1482,7 @@ public class JsonRendererTestCase {
 
     @Test
     @Timeout(300)
-    void testWsetInFields() throws IOException, InterruptedException, ExecutionException {
+    void testWsetInFields() throws InterruptedException, ExecutionException {
         Result r = new Result(new Query("/?renderer.json.jsonWsets=true"));
         var expected = dataFromSimplified(
                 "{root: { id:'toplevel', relevance:1.0, fields: { totalCount: 1 }," +
diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
index 33f840c7af0..f468d11ddf3 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
@@ -1139,6 +1139,12 @@ public class YqlParserTestCase {
         parse("select * from sources * where (default contains ({stem: false}\"m\") AND default contains ({origin: {original: \"m\'s\", offset: 0, length: 3}, andSegmenting: true}phrase(\"m\", \"s\"))) timeout 472");
     }
 
+    @Test
+    void testBuiltinOrderingFields() {
+        parse("select * from sources * where default contains 'foo' order by '[relevance]'");
+        assertEquals("[relevance]", parser.getSorting().fieldOrders().get(0).getFieldName());
+    }
+
     private void assertUrlQuery(String field, Query query, boolean startAnchor, boolean endAnchor, boolean endAnchorIsDefault) {
         boolean startAnchorIsDefault = false; // Always