From f5b26e2122a0287f37ad0c5d1b4ad95da4ac456e Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 17 Apr 2023 20:24:08 +0200 Subject: Render raw results in grouping as base64 instead of string --- container-search/abi-spec.json | 1 + .../java/com/yahoo/prelude/hitfield/RawBase64.java | 9 ++- .../yahoo/search/grouping/vespa/ResultBuilder.java | 25 +++--- .../grouping/vespa/ResultBuilderTestCase.java | 88 +++++++++++++++------- 4 files changed, 85 insertions(+), 38 deletions(-) (limited to 'container-search') diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 36531fbf5e1..50aba749570 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -209,6 +209,7 @@ ], "methods" : [ "public void (byte[])", + "public void (byte[], boolean)", "public int compareTo(com.yahoo.prelude.hitfield.RawBase64)", "public java.lang.String toString()", "public bridge synthetic int compareTo(java.lang.Object)" diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java index 2071e43f54c..ada0797ab02 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java @@ -9,8 +9,13 @@ import java.util.Base64; */ public class RawBase64 implements Comparable { private final byte[] content; + private final boolean withoutPadding; public RawBase64(byte[] content) { + this(content, false); + } + public RawBase64(byte[] content, boolean withoutPadding) { this.content = content; + this.withoutPadding = withoutPadding; } @Override @@ -20,6 +25,8 @@ public class RawBase64 implements Comparable { @Override public String toString() { - return Base64.getEncoder().encodeToString(content); + return withoutPadding + ? Base64.getEncoder().withoutPadding().encodeToString(content) + : Base64.getEncoder().encodeToString(content); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java index 7f006b098cd..2333a180690 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.vespa; +import com.yahoo.prelude.hitfield.RawBase64; import com.yahoo.search.grouping.Continuation; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.grouping.result.BoolId; @@ -28,6 +29,7 @@ import com.yahoo.searchlib.aggregation.Hit; import com.yahoo.searchlib.aggregation.HitsAggregationResult; import com.yahoo.searchlib.aggregation.MaxAggregationResult; import com.yahoo.searchlib.aggregation.MinAggregationResult; +import com.yahoo.searchlib.aggregation.RawData; import com.yahoo.searchlib.aggregation.StandardDeviationAggregationResult; import com.yahoo.searchlib.aggregation.SumAggregationResult; import com.yahoo.searchlib.aggregation.XorAggregationResult; @@ -169,7 +171,7 @@ class ResultBuilder { } else { String label = transform.getLabel(result.getTag()); if (label != null) { - group.setField(label, newResult(result, tag)); + group.setField(label, convertResult(newResult(result, tag))); } } } @@ -228,24 +230,27 @@ class ResultBuilder { return new RawId(res.getRaw()); } else if (res instanceof StringResultNode) { return new StringId(res.getString()); - } else if (res instanceof FloatBucketResultNode) { - FloatBucketResultNode bucketId = (FloatBucketResultNode)res; + } else if (res instanceof FloatBucketResultNode bucketId) { return new DoubleBucketId(bucketId.getFrom(), bucketId.getTo()); - } else if (res instanceof IntegerBucketResultNode) { - IntegerBucketResultNode bucketId = (IntegerBucketResultNode)res; + } else if (res instanceof IntegerBucketResultNode bucketId) { return new LongBucketId(bucketId.getFrom(), bucketId.getTo()); - } else if (res instanceof StringBucketResultNode) { - StringBucketResultNode bucketId = (StringBucketResultNode)res; + } else if (res instanceof StringBucketResultNode bucketId) { return new StringBucketId(bucketId.getFrom(), bucketId.getTo()); - } else if (res instanceof RawBucketResultNode) { - RawBucketResultNode bucketId = (RawBucketResultNode)res; + } else if (res instanceof RawBucketResultNode bucketId) { return new RawBucketId(bucketId.getFrom(), bucketId.getTo()); } else { throw new UnsupportedOperationException(res.getClass().getName()); } } - Object newResult(ExpressionNode execResult, int tag) { + private Object convertResult(Object value) { + if (value instanceof RawData raw) { + return new RawBase64(raw.getData(), true); + } + return value; + } + + private Object newResult(ExpressionNode execResult, int tag) { if (execResult instanceof AverageAggregationResult) { return ((AverageAggregationResult)execResult).getAverage().getNumber(); } else if (execResult instanceof CountAggregationResult) { diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java index 28a6b6a2c1d..b0b48bb8731 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java @@ -10,12 +10,39 @@ import com.yahoo.search.grouping.result.GroupList; import com.yahoo.search.grouping.result.HitList; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.Relevance; -import com.yahoo.searchlib.aggregation.*; +import com.yahoo.searchlib.aggregation.AggregationResult; +import com.yahoo.searchlib.aggregation.AverageAggregationResult; +import com.yahoo.searchlib.aggregation.CountAggregationResult; +import com.yahoo.searchlib.aggregation.ExpressionCountAggregationResult; +import com.yahoo.searchlib.aggregation.FS4Hit; +import com.yahoo.searchlib.aggregation.Group; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.searchlib.aggregation.HitsAggregationResult; +import com.yahoo.searchlib.aggregation.MaxAggregationResult; +import com.yahoo.searchlib.aggregation.MinAggregationResult; +import com.yahoo.searchlib.aggregation.SumAggregationResult; +import com.yahoo.searchlib.aggregation.XorAggregationResult; import com.yahoo.searchlib.aggregation.hll.SparseSketch; -import com.yahoo.searchlib.expression.*; +import com.yahoo.searchlib.expression.FloatBucketResultNode; +import com.yahoo.searchlib.expression.FloatResultNode; +import com.yahoo.searchlib.expression.IntegerBucketResultNode; +import com.yahoo.searchlib.expression.IntegerResultNode; +import com.yahoo.searchlib.expression.NullResultNode; +import com.yahoo.searchlib.expression.RawBucketResultNode; +import com.yahoo.searchlib.expression.RawResultNode; +import com.yahoo.searchlib.expression.ResultNode; +import com.yahoo.searchlib.expression.StringBucketResultNode; +import com.yahoo.searchlib.expression.StringResultNode; import org.junit.jupiter.api.Test; -import java.util.*; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -45,7 +72,7 @@ public class ResultBuilderTestCase { @Test void requireThatUnknownGroupIdThrows() { assertBuildFail("all(group(a) each(output(count())))", - Arrays.asList(newGrouping(new Group().setTag(2).setId(new MyResultNode()))), + List.of(newGrouping(new Group().setTag(2).setId(new MyResultNode()))), "com.yahoo.search.grouping.vespa.ResultBuilderTestCase$MyResultNode"); } @@ -60,10 +87,18 @@ public class ResultBuilderTestCase { assertResult("69", new ExpressionCountAggregationResult(new SparseSketch(), sketch -> 69)); } + @Test + void requireThatAllBasicResultsCanBeConverted() { + assertResult("69", new MinAggregationResult(new IntegerResultNode(69))); + assertResult("69.3", new MinAggregationResult(new FloatResultNode(69.3))); + assertResult("69.6", new MinAggregationResult(new StringResultNode("69.6"))); + assertResult("Bgk", new MinAggregationResult(new RawResultNode(new byte[]{6,9}))); + } + @Test void requireThatUnknownExpressionNodeThrows() { assertBuildFail("all(group(a) each(output(count())))", - Arrays.asList(newGrouping(newGroup(2, 2, new MyAggregationResult().setTag(3)))), + List.of(newGrouping(newGroup(2, 2, new MyAggregationResult().setTag(3)))), "com.yahoo.search.grouping.vespa.ResultBuilderTestCase$MyAggregationResult"); } @@ -127,10 +162,10 @@ public class ResultBuilderTestCase { @Test void requireThatParallelResultsAreTransformed() { assertBuild("all(group(foo) each(output(count())) as(bar) each(output(count())) as(baz))", - Arrays.asList(new Grouping().setRoot(newGroup(1, 0)), + List.of(new Grouping().setRoot(newGroup(1, 0)), new Grouping().setRoot(newGroup(1, 0)))); assertBuildFail("all(group(foo) each(output(count())) as(bar) each(output(count())) as(baz))", - Arrays.asList(new Grouping().setRoot(newGroup(2)), + List.of(new Grouping().setRoot(newGroup(2)), new Grouping().setRoot(newGroup(3))), "Expected 1 group, got 2."); } @@ -138,15 +173,15 @@ public class ResultBuilderTestCase { @Test void requireThatTagsAreHandledCorrectly() { assertBuild("all(group(a) each(output(count())))", - Arrays.asList(newGrouping( + List.of(newGrouping( newGroup(7, new CountAggregationResult(0))))); } @Test void requireThatEmptyBranchesArePruned() { - assertBuildFail("all()", Collections.emptyList(), "Expected 1 group, got 0."); - assertBuildFail("all(group(a))", Collections.emptyList(), "Expected 1 group, got 0."); - assertBuildFail("all(group(a) each())", Collections.emptyList(), "Expected 1 group, got 0."); + assertBuildFail("all()", List.of(), "Expected 1 group, got 0."); + assertBuildFail("all(group(a))", List.of(), "Expected 1 group, got 0."); + assertBuildFail("all(group(a) each())", List.of(), "Expected 1 group, got 0."); Grouping grouping = newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))); String expectedOutput = "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=69}[]]]"; @@ -189,14 +224,14 @@ public class ResultBuilderTestCase { "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); assertLayout("all(group(foo) each(each(output(summary())) as(bar)" + " each(output(summary())) as(baz)))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 2))), + List.of(newGrouping(newGroup(2, newHitList(3, 2))), newGrouping(newGroup(2, newHitList(4, 2)))), "RootGroup{id=group:root}[GroupList{label=foo}[Group{id=group:2}[" + "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}], " + "HitList{label=baz}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); assertLayout("all(group(foo) each(each(output(summary())))" + " each(each(output(summary()))) as(bar))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 2))), + List.of(newGrouping(newGroup(2, newHitList(3, 2))), newGrouping(newGroup(4, newHitList(5, 2)))), "RootGroup{id=group:root}[" + "GroupList{label=foo}[Group{id=group:2}[HitList{label=hits}[Hit{id=hit:1}, Hit{id=hit:2}]]], " + @@ -273,18 +308,18 @@ public class ResultBuilderTestCase { assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + " each(output(count())) as(bar))", - Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + List.of(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), "[]"); assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + " each(output(count())) as(bar))", - Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + List.of(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), newOffset(newResultId(0), 2, 1), "[0=1]"); assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + " each(output(count())) as(bar))", - Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + List.of(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), newComposite(newOffset(newResultId(0), 2, 2), newOffset(newResultId(1), 4, 1)), @@ -299,18 +334,18 @@ public class ResultBuilderTestCase { assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + " each(max(2) each(output(summary()))) as(bar))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + List.of(newGrouping(newGroup(2, newHitList(3, 4))), newGrouping(newGroup(4, newHitList(5, 4)))), "[]"); assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + " each(max(2) each(output(summary()))) as(bar))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + List.of(newGrouping(newGroup(2, newHitList(3, 4))), newGrouping(newGroup(4, newHitList(5, 4)))), newOffset(newResultId(0, 0, 0), 3, 1), "[0.0.0=1]"); assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + " each(max(2) each(output(summary()))) as(bar))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + List.of(newGrouping(newGroup(2, newHitList(3, 4))), newGrouping(newGroup(4, newHitList(5, 4)))), newComposite(newOffset(newResultId(0, 0, 0), 3, 2), newOffset(newResultId(1, 0, 0), 5, 1)), @@ -404,7 +439,7 @@ public class ResultBuilderTestCase { void requireThatGroupListContinuationsCanBeSetInSiblingGroupLists() { String request = "all(group(a) max(2) each(output(count())) as(foo)" + " each(output(count())) as(bar))"; - List result = Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + List result = List.of(newGrouping(newGroup(2, 1, new CountAggregationResult(1)), newGroup(2, 2, new CountAggregationResult(2)), newGroup(2, 3, new CountAggregationResult(3)), newGroup(2, 4, new CountAggregationResult(4))), @@ -646,7 +681,7 @@ public class ResultBuilderTestCase { void requireThatHitListContinuationsCanBeSetInSiblingHitLists() { String request = "all(group(a) each(max(2) each(output(summary()))) as(foo)" + " each(max(2) each(output(summary()))) as(bar))"; - List result = Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + List result = List.of(newGrouping(newGroup(2, newHitList(3, 4))), newGrouping(newGroup(4, newHitList(5, 4)))); assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 0), newOffset(newResultId(1, 0, 0), 5, 5)), @@ -839,7 +874,7 @@ public class ResultBuilderTestCase { } private static void assertResultCont(String request, Grouping result, Continuation cont, String expected) { - assertOutput(request, Arrays.asList(result), cont, new ResultContWriter(), expected); + assertOutput(request, List.of(result), cont, new ResultContWriter(), expected); } private static void assertResultCont(String request, List result, String expected) { @@ -851,11 +886,11 @@ public class ResultBuilderTestCase { } private static void assertContinuation(String request, Grouping result, String expected) { - assertOutput(request, Arrays.asList(result), null, new ContinuationWriter(), expected); + assertOutput(request, List.of(result), null, new ContinuationWriter(), expected); } private static void assertContinuation(String request, Grouping result, Continuation cont, String expected) { - assertOutput(request, Arrays.asList(result), cont, new ContinuationWriter(), expected); + assertOutput(request, List.of(result), cont, new ContinuationWriter(), expected); } private static void assertContinuation(String request, List result, Continuation cont, String expected) { @@ -863,7 +898,7 @@ public class ResultBuilderTestCase { } private static void assertLayout(String request, Grouping result, String expected) { - assertOutput(request, Arrays.asList(result), null, new LayoutWriter(), expected); + assertOutput(request, List.of(result), null, new LayoutWriter(), expected); } private static void assertLayout(String request, List result, String expected) { @@ -953,8 +988,7 @@ public class ResultBuilderTestCase { } String toString(Continuation cnt) { - if (cnt instanceof OffsetContinuation) { - OffsetContinuation off = (OffsetContinuation)cnt; + if (cnt instanceof OffsetContinuation off) { String id = off.getResultId().toString().replace(", ", "."); return id.substring(5, id.length() - 1) + "=" + off.getOffset(); } else if (cnt instanceof CompositeContinuation) { -- cgit v1.2.3