From 23ff22b8c9e5e0333c45699cef89404bb74fe381 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 24 Mar 2023 09:57:48 +0100 Subject: Add streamed fill test skeleton --- .../java/com/yahoo/prelude/hitfield/HitField.java | 27 ++++------ .../com/yahoo/prelude/hitfield/XmlRenderer.java | 10 ++-- .../com/yahoo/search/rendering/JsonRenderer.java | 2 +- .../com/yahoo/search/handler/StreamedFillTest.java | 60 ++++++++++++++++++++++ .../rendering/AsyncGroupPopulationTestCase.java | 2 +- .../search/rendering/JsonRendererTestCase.java | 2 +- .../com/yahoo/search/yql/YqlParserTestCase.java | 6 +++ 7 files changed, 82 insertions(+), 27 deletions(-) create mode 100644 container-search/src/test/java/com/yahoo/search/handler/StreamedFillTest.java diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java index 5d26a71137e..8c6f3df7ca1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java @@ -21,7 +21,7 @@ public class HitField { private final String rawContent; private final boolean isCJK; - private boolean xmlProperty; + private final boolean xmlProperty; private List tokenizedContent = null; private String content; @@ -191,8 +191,8 @@ public class HitField { private List tokenizePretokenized() { String[] pre = rawContent.split("\u001F+"); List tokenized = new ArrayList<>(pre.length); - for (int i = 0; i < pre.length; i++) { - tokenized.add(createToken(pre[i], true)); + for (String s : pre) { + tokenized.add(createToken(s, true)); } return tokenized; } @@ -245,9 +245,7 @@ public class HitField { */ public void setTokenizedContent(List list) { tokenizedContent = new ArrayList<>(list.size()); - for (Iterator i = list.iterator(); i.hasNext(); ) { - tokenizedContent.add(i.next()); - } + tokenizedContent.addAll(list); // Must null content reference _before_ calling getContent() content = null; } @@ -262,9 +260,8 @@ public class HitField { public String getContent() { if (content == null) { StringBuilder buf = new StringBuilder(); - Iterator iter = ensureTokenized().iterator(); - while(iter.hasNext()) { - buf.append(iter.next().getContent()); + for (FieldPart fieldPart : ensureTokenized()) { + buf.append(fieldPart.getContent()); } content = buf.toString(); } @@ -274,9 +271,7 @@ public class HitField { /** Returns the content of this field, using the arguments as bolding tags */ public String getContent(String boldOpenTag, String boldCloseTag, String separatorTag) { StringBuilder b = new StringBuilder(); - Iterator iter = ensureTokenized().iterator(); - while(iter.hasNext()) { - FieldPart f = iter.next(); + for (FieldPart f : ensureTokenized()) { if (f instanceof BoldOpenFieldPart && boldOpenTag != null && boldOpenTag.length() > 0) b.append(boldOpenTag); else if (f instanceof BoldCloseFieldPart && boldCloseTag != null && boldCloseTag.length() > 0) @@ -316,9 +311,7 @@ public class HitField { String separatorTag, boolean inAttribute) { StringBuilder xml = new StringBuilder(); - Iterator iter = ensureTokenized().iterator(); - while(iter.hasNext()) { - FieldPart f = iter.next(); + for (FieldPart f : ensureTokenized()) { if (f instanceof BoldOpenFieldPart && boldOpenTag != null && boldOpenTag.length() > 0) @@ -344,9 +337,7 @@ public class HitField { */ public String bareContent(boolean XMLQuote, boolean inAttribute) { StringBuilder bareContent = new StringBuilder(); - Iterator iter = ensureTokenized().iterator(); - while(iter.hasNext()) { - FieldPart f = iter.next(); + for (FieldPart f : ensureTokenized()) { if (f instanceof MarkupFieldPart) continue; diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java index 09dafe880f0..b55a4ff9b71 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/XmlRenderer.java @@ -44,10 +44,10 @@ public class XmlRenderer { byte[] data = value.asData(); renderTarget.append(""); - for (int i = 0; i < data.length; i++) { + for (byte datum : data) { for (int sh = 4; sh >= 0; sh -= 4) { - int val = (data[i] >> sh) & 0xF; - char hexdigit = (val < 10) ? ((char)('0' + val)) : ((char)('A' + val - 10)); + int val = (datum >> sh) & 0xF; + char hexdigit = (val < 10) ? ((char) ('0' + val)) : ((char) ('A' + val - 10)); renderTarget.append(hexdigit); } } @@ -84,9 +84,7 @@ public class XmlRenderer { } private void indent(int nestingLevel) { - for (int i = 0; i < nestingLevel; ++i) { - renderTarget.append(" "); - } + renderTarget.append(" ".repeat(Math.max(0, nestingLevel))); } private void renderMap(Inspector sequence, int nestingLevel) { diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 44620179c1d..05df0883f07 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -422,7 +422,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer { protected void renderGroupMetadata(GroupId id) throws IOException { if (!(id instanceof ValueGroupId || id instanceof BucketGroupId)) return; - if (id instanceof ValueGroupId valueId) { + if (id instanceof ValueGroupId valueId) { generator.writeStringField(GROUPING_VALUE, getIdValue(valueId)); } else { BucketGroupId bucketId = (BucketGroupId) id; diff --git a/container-search/src/test/java/com/yahoo/search/handler/StreamedFillTest.java b/container-search/src/test/java/com/yahoo/search/handler/StreamedFillTest.java new file mode 100644 index 00000000000..bea16fbbdbc --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/handler/StreamedFillTest.java @@ -0,0 +1,60 @@ +package com.yahoo.search.handler; + +import ai.vespa.cloud.ZoneInfo; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.core.ContainerHttpConfig; +import com.yahoo.container.handler.threadpool.ContainerThreadPool; +import com.yahoo.container.jdisc.ContentChannelOutputStream; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.jdisc.handler.ReadableContentChannel; +import com.yahoo.jdisc.test.MockMetric; +import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; +import com.yahoo.search.searchchain.ExecutionFactory; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Tests filling hits as they are rendered to avoid having to keep all hits in memory. + * + * @author bratseth + */ +public class StreamedFillTest { + + @Test + void testStreamedFill() { + var handler = new SearchHandler(new MockMetric(), + new SimpleContainerThreadpool(), + new CompiledQueryProfileRegistry(), + new ContainerHttpConfig.Builder().build(), + new ComponentRegistry(), + ExecutionFactory.empty(), + ZoneInfo.defaultInfo()); + var request = HttpRequest.createTestRequest("?query=foo", com.yahoo.jdisc.http.HttpRequest.Method.GET); + var response = (HttpSearchResponse)handler.handle(request); + System.out.println(render(response)); + } + + private String render(HttpSearchResponse response) { + try { + var out = new ReadableContentChannel(); + response.render(new ContentChannelOutputStream(out), out, null); + return new String(out.toStream().readAllBytes(), UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static class SimpleContainerThreadpool implements ContainerThreadPool { + + private final Executor executor = Executors.newCachedThreadPool(); + + @Override public Executor executor() { return executor; } + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java index c038b3c67c9..f51b2c2a216 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/AsyncGroupPopulationTestCase.java @@ -31,7 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test adding hits to a hit group during rendering. * - * @author Steinar Knutsen + * @author Steinar Knutsen */ public class AsyncGroupPopulationTestCase { private static class WrappedFuture extends CompletableFuture { diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index 6db3ec9735e..9cc2b4e6227 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -1482,7 +1482,7 @@ public class JsonRendererTestCase { @Test @Timeout(300) - void testWsetInFields() throws IOException, InterruptedException, ExecutionException { + void testWsetInFields() throws InterruptedException, ExecutionException { Result r = new Result(new Query("/?renderer.json.jsonWsets=true")); var expected = dataFromSimplified( "{root: { id:'toplevel', relevance:1.0, fields: { totalCount: 1 }," + diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 33f840c7af0..f468d11ddf3 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -1139,6 +1139,12 @@ public class YqlParserTestCase { parse("select * from sources * where (default contains ({stem: false}\"m\") AND default contains ({origin: {original: \"m\'s\", offset: 0, length: 3}, andSegmenting: true}phrase(\"m\", \"s\"))) timeout 472"); } + @Test + void testBuiltinOrderingFields() { + parse("select * from sources * where default contains 'foo' order by '[relevance]'"); + assertEquals("[relevance]", parser.getSorting().fieldOrders().get(0).getFieldName()); + } + private void assertUrlQuery(String field, Query query, boolean startAnchor, boolean endAnchor, boolean endAnchorIsDefault) { boolean startAnchorIsDefault = false; // Always -- cgit v1.2.3