Merge pull request #24052 from vespa-engine/geirst/support-array-of-string-when-converting-highlighting

Handle array of string fields when converting the backend binary high…

2 files changed, 61 insertions, 7 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java
index 44c490b8ed1..4fc3f7919cf 100644
--- a/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JuniperSearcher.java
@@ -11,6 +11,11 @@ import com.yahoo.component.ComponentId;
 import com.yahoo.component.chain.dependencies.After;
 import com.yahoo.component.chain.dependencies.Before;
 import com.yahoo.component.chain.dependencies.Provides;
+import com.yahoo.data.access.ArrayTraverser;
+import com.yahoo.data.access.Inspectable;
+import com.yahoo.data.access.Inspector;
+import com.yahoo.data.access.Type;
+import com.yahoo.data.access.simple.Value;
 import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.search.Searcher;
@@ -104,14 +109,51 @@ public class JuniperSearcher extends Searcher {
 
             for (Index index : indexFacts.getIndexes(searchDefinitionField.toString())) {
                 if (index.getDynamicSummary() || index.getHighlightSummary()) {
-                    HitField fieldValue = fastHit.buildHitField(index.getName(), true);
-                    if (fieldValue != null)
-                        insertTags(fieldValue, bolding, index.getDynamicSummary());
+                    var field = fastHit.getField(index.getName());
+                    if (StringArrayConverter.shouldHandleField(field)) {
+                        new StringArrayConverter(fastHit, index, field, bolding);
+                    } else {
+                        HitField fieldValue = fastHit.buildHitField(index.getName(), true);
+                        if (fieldValue != null) {
+                            insertTags(fieldValue, bolding, index.getDynamicSummary());
+                        }
+                    }
                 }
             }
         }
     }
 
+    private class StringArrayConverter implements ArrayTraverser {
+
+        private Index index;
+        private boolean bolding;
+        private Value.ArrayValue convertedField = new Value.ArrayValue();
+
+        /**
+         * This converts the backend binary highlighting of each item in an array of string field,
+         * and creates a new field that replaces the original.
+         */
+        StringArrayConverter(FastHit hit, Index index, Object field, boolean bolding) {
+            this.index = index;
+            this.bolding = bolding;
+            ((Inspectable)field).inspect().traverse(this);
+            hit.setField(index.getName(), convertedField);
+        }
+
+        static boolean shouldHandleField(Object field) {
+            return (field instanceof Inspectable) &&
+                    (((Inspectable)field).inspect().type() == Type.ARRAY);
+        }
+
+        @Override
+        public void entry(int idx, Inspector inspector) {
+            // This is how HitField is instantiated in Hit.buildHitField() when forceNoPreTokenize=true.
+            var hitField = new HitField(index.getName(), inspector.asString(), false);
+            insertTags(hitField, bolding, index.getDynamicSummary());
+            convertedField.add(hitField.getContent());
+        }
+    }
+
     private void insertTags(HitField field, boolean bolding, boolean dynteaser) {
         boolean insideHighlight = false;
         for (ListIterator<FieldPart> i = field.listIterator(); i.hasNext();) {
diff --git a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java
index bd72ed9fd08..d374bfdeb7b 100644
--- a/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/searcher/test/JuniperSearcherTestCase.java
@@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList;
 import com.yahoo.component.ComponentId;
 import com.yahoo.component.chain.Chain;
 import com.yahoo.container.QrSearchersConfig;
+import com.yahoo.data.access.simple.Value;
 import com.yahoo.prelude.Index;
 import com.yahoo.prelude.IndexFacts;
 import com.yahoo.prelude.IndexModel;
@@ -40,7 +41,7 @@ public class JuniperSearcherTestCase {
      * @param sdName the search definition type of the returned hit
      * @param content the content of the "dynteaser" field of the returned hit
      */
-    private Chain<Searcher> createSearchChain(String sdName, String content) {
+    private Chain<Searcher> createSearchChain(String sdName, Object content) {
         JuniperSearcher searcher = new JuniperSearcher(new ComponentId("test"),
                                                        new QrSearchersConfig(new QrSearchersConfig.Builder()));
 
@@ -50,7 +51,7 @@ public class JuniperSearcherTestCase {
         return new Chain<>(searcher, docsource);
     }
 
-    private void addResult(Query query, String sdName, String content, DocumentSourceSearcher docsource) {
+    private void addResult(Query query, String sdName, Object content, DocumentSourceSearcher docsource) {
         Result r = new Result(query);
         FastHit hit = new FastHit();
         hit.setId("http://abc.html");
@@ -62,7 +63,7 @@ public class JuniperSearcherTestCase {
     }
 
     /** Creates a result of the search definiton "one" */
-    private Result createResult(String content) {
+    private Result createResult(Object content) {
         return createResult("one", content, true);
     }
 
@@ -70,7 +71,7 @@ public class JuniperSearcherTestCase {
         return createResult(sdName, content, true);
     }
 
-    private Result createResult(String sdName, String content, boolean bolding) {
+    private Result createResult(String sdName, Object content, boolean bolding) {
         Chain<Searcher> chain = createSearchChain(sdName, content);
         Query query = new Query("?query=12");
         if ( ! bolding)
@@ -123,6 +124,17 @@ public class JuniperSearcherTestCase {
     }
 
     @Test
+    void test_field_rewriting_for_array_of_string_field() {
+        var content = new Value.ArrayValue()
+                .add("\u001Faaa\u001F\u001Ebbb\u001Fccc\u001Fddd")
+                .add("\u001Feee\u001F\u001Efff\u001Fggg\u001Fhhh");
+        Result check = createResult(content);
+        assertEquals(1, check.getHitCount());
+        assertEquals("[\"<hi>aaa</hi><sep />bbb<hi>ccc</hi>ddd\",\"<hi>eee</hi><sep />fff<hi>ggg</hi>hhh\"]",
+                check.hits().get(0).getField("dynteaser").toString());
+    }
+
+    @Test
     void testNoRewritingDueToSearchDefinition() {
         Result check = createResult("two", "\u001FXYZ\u001F\u001EQWE\u001FJKL\u001FASD&");
         assertEquals(1, check.getHitCount());