add option to recognize maps deep inside hit fields

3 files changed, 111 insertions, 7 deletions

diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index b7ce40f19a2..c90342cf60b 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -7075,6 +7075,7 @@
     "methods": [
       "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean)",
       "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean, boolean)",
+      "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean, boolean, boolean)",
       "public void accept(java.lang.String, java.lang.Object)",
       "public void accept(java.lang.String, byte[], int, int)",
       "protected boolean shouldRender(java.lang.String, java.lang.Object)",
diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
index ee0e7f4fe0e..521c454467f 100644
--- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
+++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java
@@ -76,6 +76,7 @@ import static com.fasterxml.jackson.databind.SerializationFeature.FLUSH_AFTER_WR
 // NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc.
 public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
 
+    private static final CompoundName WRAP_ALL_MAPS = new CompoundName("renderer.json.wrapAllMaps");
     private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug");
     private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback");
     private static final CompoundName TENSOR_FORMAT = new CompoundName("format.tensors");
@@ -125,6 +126,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
     private FieldConsumer fieldConsumer;
     private Deque<Integer> renderedChildren;
     private boolean debugRendering;
+    private boolean wrapAllMaps;
     private LongSupplier timeSource;
     private OutputStream stream;
 
@@ -159,6 +161,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
     public void init() {
         super.init();
         debugRendering = false;
+        wrapAllMaps = false;
         setGenerator(null, debugRendering);
         renderedChildren = null;
         timeSource = System::currentTimeMillis;
@@ -169,6 +172,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
     public void beginResponse(OutputStream stream) throws IOException {
         beginJsonCallback(stream);
         debugRendering = getDebugRendering(getResult().getQuery());
+        wrapAllMaps = getWrapAllMaps(getResult().getQuery());
         tensorShortFormRendering = getTensorShortFormRendering(getResult().getQuery());
         setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering);
         renderedChildren = new ArrayDeque<>();
@@ -200,6 +204,10 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
         generator.writeEndObject();
     }
 
+    private boolean getWrapAllMaps(Query q) {
+        return q != null && q.properties().getBoolean(WRAP_ALL_MAPS, false);
+    }
+
     private boolean getDebugRendering(Query q) {
         return q != null && q.properties().getBoolean(DEBUG_RENDERING_KEY, false);
     }
@@ -514,11 +522,15 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
 
     private void setGenerator(JsonGenerator generator, boolean debugRendering) {
         this.generator = generator;
-        this.fieldConsumer = generator == null ? null : createFieldConsumer(generator, debugRendering);
+        this.fieldConsumer = generator == null ? null : createFieldConsumer(generator, debugRendering, wrapAllMaps);
     }
 
     protected FieldConsumer createFieldConsumer(JsonGenerator generator, boolean debugRendering) {
-        return new FieldConsumer(generator, debugRendering, tensorShortFormRendering);
+        return createFieldConsumer(generator, debugRendering, this.wrapAllMaps);
+    }
+
+    private FieldConsumer createFieldConsumer(JsonGenerator generator, boolean debugRendering, boolean wrapAllMaps) {
+        return new FieldConsumer(generator, debugRendering, tensorShortFormRendering, wrapAllMaps);
     }
 
     /**
@@ -537,6 +549,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
 
         private final JsonGenerator generator;
         private final boolean debugRendering;
+        private final boolean wrapAllMaps;
         private final boolean tensorShortForm;
 
         private MutableBoolean hasFieldsField;
@@ -544,11 +557,14 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
         public FieldConsumer(JsonGenerator generator, boolean debugRendering) {
             this(generator, debugRendering, false);
         }
-
         public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm) {
+            this(generator, debugRendering, tensorShortForm, false);
+        }
+        public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm, boolean wrapAllMaps) {
             this.generator = generator;
             this.debugRendering = debugRendering;
             this.tensorShortForm = tensorShortForm;
+            this.wrapAllMaps = wrapAllMaps;
         }
 
         /**
@@ -618,6 +634,48 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
             return true;
         }
 
+        private static Inspector deepWrapAsMap(Inspector data) {
+            System.err.println("deep wrap: "+data);
+            if (data.type() == Type.ARRAY) {
+                var map = new Value.ObjectValue();
+                for (int i = 0; i < data.entryCount(); i++) {
+                    Inspector obj = data.entry(i);
+                    if (map != null && obj.type() == Type.OBJECT && obj.fieldCount() == 2) {
+                        Inspector key = obj.field("key");
+                        Inspector value = obj.field("value");
+                        if (key.type() == Type.STRING && value.valid()) {
+                            map.put(key.asString(), deepWrapAsMap(value));
+                        } else {
+                            map = null;
+                        }
+                    } else {
+                        map = null;
+                    }
+                }
+                if (map != null) {
+                    System.err.println("recognized as map -> "+map);
+                    return map;
+                }
+                var array = new Value.ArrayValue();
+                for (int i = 0; i < data.entryCount(); i++) {
+                    Inspector obj = data.entry(i);
+                    array.add(deepWrapAsMap(obj));
+                }
+                System.err.println("just an array -> "+array);
+                return array;
+            }
+            if (data.type() == Type.OBJECT) {
+                var object = new Value.ObjectValue();
+                for (var entry : data.fields()) {
+                    object.put(entry.getKey(), deepWrapAsMap(entry.getValue()));
+                }
+                System.err.println("just an object -> "+object);
+                return object;
+            }
+            System.err.println("just data");
+            return data;
+        }
+
         private static Inspector wrapAsMap(Inspector data) {
             if (data.type() != Type.ARRAY) return null;
             if (data.entryCount() == 0) return null;
@@ -636,12 +694,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> {
         }
 
         private void renderInspector(Inspector data) throws IOException {
-            Inspector asMap = wrapAsMap(data);
+            Inspector asMap = wrapAllMaps ? deepWrapAsMap(data) : wrapAsMap(data);
             if (asMap != null) {
-                StringBuilder intermediate = new StringBuilder();
-                JsonRender.render(asMap, intermediate, true);
-                generator.writeRawValue(intermediate.toString());
+                System.err.println("maybe converted: "+asMap);
+                renderInspectorDirect(asMap);
             } else {
+                System.err.println("not converted: "+data);
                 renderInspectorDirect(data);
             }
         }
diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
index 290b7266a3a..9be2945e67b 100644
--- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java
@@ -1259,6 +1259,51 @@ public class JsonRendererTestCase {
         assertEqualJson(expected, summary);
     }
 
+    private static SlimeAdapter dataFromSimplified(String simplified) {
+        var decoder = new com.yahoo.slime.JsonDecoder();
+        var slime = decoder.decode(new Slime(), Utf8.toBytes(simplified));
+        return new SlimeAdapter(slime.get());
+    }
+
+    @Test
+    public void testMapDeepInFields() throws IOException, InterruptedException, ExecutionException {
+        var expected = dataFromSimplified(
+                "{'root':{'id':'toplevel','relevance':1.0,'fields':{'totalCount':1}," +
+                "  'children': [ { 'id': 'myHitName', 'relevance': 1.0," +
+                "    'fields': { " +
+                "      'f1': [ 'v1', { 'mykey1': 'myvalue1', 'mykey2': 'myvalue2' } ]," +
+                "      'f2': { 'i1': 'v2', 'i2': { 'mykey3': 'myvalue3' }, 'i3': 'v3' }," +
+                "      'f3': { 'a': 42, 'b': 17.75, 'c': [ 'v4', 'v5' ] }" +
+                "    }" +
+                "  } ]" +
+                "}}");
+        Result r = new Result(new Query("/?renderer.json.wrapAllMaps=true"));
+        Hit h = new Hit("myHitName");
+        h.setField("f1", dataFromSimplified("[ 'v1', [ { 'key': 'mykey1', 'value': 'myvalue1' }, { 'key': 'mykey2', 'value': 'myvalue2' } ] ]"));
+        h.setField("f2", dataFromSimplified("{ 'i1': 'v2', 'i2': [ { 'key': 'mykey3', 'value': 'myvalue3' } ], 'i3': 'v3' }"));
+        h.setField("f3", dataFromSimplified("{ 'a': 42, 'b': 17.75, 'c': [ 'v4', 'v5' ] }"));
+        r.hits().add(h);
+        r.setTotalHitCount(1L);
+        String summary = render(r);
+        assertEqualJson(expected.toString(), summary);
+
+        expected = dataFromSimplified(
+                "{'root':{'id':'toplevel','relevance':1.0,'fields':{'totalCount':1}," +
+                "  'children': [ { 'id': 'myHitName', 'relevance': 1.0," +
+                "    'fields': { " +
+                "      'f1': [ 'v1', [ { 'key': 'mykey1', 'value': 'myvalue1' }, { 'key': 'mykey2', 'value': 'myvalue2' } ] ]," +
+                "      'f2': { 'i1': 'v2', 'i2': [ { 'key': 'mykey3', 'value': 'myvalue3' } ], 'i3': 'v3' }," +
+                "      'f3': { 'a': 42, 'b': 17.75, 'c': [ 'v4', 'v5' ] }" +
+                "    }" +
+                "  } ]" +
+                "}}");
+        r = new Result(new Query("/?renderer.json.wrapAllMaps=false"));
+        r.hits().add(h);
+        r.setTotalHitCount(1L);
+        summary = render(r);
+        assertEqualJson(expected.toString(), summary);
+    }
+
     @Test
     public void testThatTheJsonValidatorCanCatchErrors() {
         String json = "{"