Prepare for raw fields to be presented as base64 encoded in summary.

10 files changed, 117 insertions, 7 deletions

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
index 64c5590b689..9b7434dccab 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java
@@ -61,6 +61,8 @@ public class Search implements ImmutableSearch {
     /** True if this doesn't define a search, just a document type */
     private boolean documentsOnly = false;
 
+    private boolean rawAsBase64 = false;
+
     /** The stemming setting of this search definition. Default is BEST. */
     private Stemming stemming = Stemming.BEST;
 
@@ -125,6 +127,16 @@ public class Search implements ImmutableSearch {
     }
 
     /**
+     * Returns true if 'raw' fields shall be presented as base64 in summary
+     * Note that tis is temporary and will disappear on Vespa 8 as it will become default, and only option.
+     *
+     * @return true if raw shall be encoded as base64 in summary
+     */
+    public boolean isRawAsBase64() { return rawAsBase64; }
+
+    public void enableRawAsBase64() { rawAsBase64 = true; }
+
+    /**
      * Sets the stemming default of fields. Default is ALL
      *
      * @param stemming set default stemming for this searchdefinition
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
index b08e9948ecd..af2168545dc 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
@@ -27,6 +27,7 @@ public class SummaryClass extends Derived {
 
     /** True if this summary class needs to access summary information on disk */
     private boolean accessingDiskSummary = false;
+    private final boolean rawAsBase64;
 
     /** The summary fields of this indexed by name */
     private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>();
@@ -42,6 +43,7 @@ public class SummaryClass extends Derived {
      */
     public SummaryClass(Search search, DocumentSummary summary, DeployLogger deployLogger) {
         this.deployLogger = deployLogger;
+        this.rawAsBase64 = search.isRawAsBase64();
         deriveName(summary);
         deriveFields(search,summary);
         deriveImplicitFields(summary);
@@ -74,12 +76,12 @@ public class SummaryClass extends Derived {
     private void addField(String name, DataType type, SummaryTransform transform) {
         if (fields.containsKey(name)) {
             SummaryClassField sf = fields.get(name);
-            if (!SummaryClassField.convertDataType(type, transform).equals(sf.getType())) {
+            if (!SummaryClassField.convertDataType(type, transform, rawAsBase64).equals(sf.getType())) {
                 deployLogger.logApplicationPackage(Level.WARNING, "Conflicting definition of field " + name + ". " +
                                "Declared as type " + sf.getType() + " and " + type);
             }
         } else {
-            fields.put(name, new SummaryClassField(name, type, transform));
+            fields.put(name, new SummaryClassField(name, type, transform, rawAsBase64));
         }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java
index c0093e031d1..3c29971a74c 100644
--- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java
+++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java
@@ -44,6 +44,7 @@ public class SummaryClassField {
         DOUBLE("double"),
         STRING("string"),
         DATA("data"),
+        RAW("raw"),
         LONGSTRING("longstring"),
         LONGDATA("longdata"),
         XMLSTRING("xmlstring"),
@@ -67,9 +68,9 @@ public class SummaryClassField {
         }
     }
 
-    public SummaryClassField(String name, DataType type, SummaryTransform transform) {
+    public SummaryClassField(String name, DataType type, SummaryTransform transform, boolean rawAsBase64) {
         this.name = name;
-        this.type = convertDataType(type, transform);
+        this.type = convertDataType(type, transform, rawAsBase64);
     }
 
     public String getName() { return name; }
@@ -77,7 +78,7 @@ public class SummaryClassField {
     public Type getType() { return type; }
 
     /** Converts to the right summary field type from a field datatype and a transform*/
-    public static Type convertDataType(DataType fieldType, SummaryTransform transform) {
+    public static Type convertDataType(DataType fieldType, SummaryTransform transform, boolean rawAsBase64) {
         FieldValue fval = fieldType.createFieldValue();
         if (fval instanceof StringFieldValue) {
             if (transform != null && transform.equals(SummaryTransform.RANKFEATURES)) {
@@ -102,7 +103,7 @@ public class SummaryClassField {
         } else if (fval instanceof ByteFieldValue) {
             return Type.BYTE;
         } else if (fval instanceof Raw) {
-            return Type.DATA;
+            return rawAsBase64 ? Type.RAW : Type.DATA;
         } else if (fval instanceof Struct) {
             return Type.JSONSTRING;
         } else if (fval instanceof PredicateFieldValue) {
diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj
index 88c45a18e4e..8f99c388fb1 100644
--- a/config-model/src/main/javacc/SDParser.jj
+++ b/config-model/src/main/javacc/SDParser.jj
@@ -237,6 +237,7 @@ TOKEN :
 | < RANKPROFILE: "rank-profile" >
 | < RANKDEGRADATIONFREQ: "rank-degradation-frequency" >
 | < RANKDEGRADATION: "rank-degradation" >
+| < RAW_AS_BASE64_IN_SUMMARY: "raw-as-base64-in-summary" >
 | < RPBINSIZE: "doc-frequency" >
 | < RPBINLOW:  "min-fullrank-docs">
 | < RPPOSBINSIZE: "occurrences-per-doc" >
@@ -451,6 +452,7 @@ Search rootSchema(String dir) :
 Object rootSchemaItem(Search search) : { }
 {
     ( document(search)
+      | rawAsBase64(search)
       | documentSummary(search)
       | field(null, search)
       | index(search, null)
@@ -561,6 +563,12 @@ Object documentBody(SDDocumentType document, Search search) :
     { return null; }
 }
 
+void rawAsBase64(Search search) :
+{}
+{
+    <RAW_AS_BASE64_IN_SUMMARY> { search.enableRawAsBase64(); }
+}
+
 /**
  * Consumes a document head block.
  *
diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
index afbc9f52f6b..bfc738a4f87 100644
--- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
+++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java
@@ -23,6 +23,37 @@ import static org.junit.Assert.assertNull;
 public class SummaryTestCase extends SchemaTestCase {
 
     @Test
+    public void deriveRawAsBase64() throws ParseException {
+        String sd = joinLines(
+                "schema s {",
+                "  raw-as-base64-in-summary",
+                "  document s {",
+                "      field raw_field type raw {",
+                "          indexing: summary",
+                "      }",
+                "  }",
+                "}");
+        Search search = SearchBuilder.createFromString(sd).getSearch();
+        SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger());
+        assertEquals(SummaryClassField.Type.RAW, summary.getField("raw_field").getType());
+    }
+
+    @Test
+    public void deriveRawAsLegacy() throws ParseException {
+        String sd = joinLines(
+                "schema s {",
+                "  document s {",
+                "      field raw_field type raw {",
+                "          indexing: summary",
+                "      }",
+                "  }",
+                "}");
+        Search search = SearchBuilder.createFromString(sd).getSearch();
+        SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger());
+        assertEquals(SummaryClassField.Type.DATA, summary.getField("raw_field").getType());
+    }
+
+    @Test
     public void testDeriving() throws IOException, ParseException {
         Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd");
         SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger());
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index 74ed9d33f04..b577660c1b9 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -201,6 +201,18 @@
     ],
     "fields": []
   },
+  "com.yahoo.prelude.hitfield.RawBase64": {
+    "superClass": "java.lang.Object",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public void <init>(byte[])",
+      "public java.lang.String toString()"
+    ],
+    "fields": []
+  },
   "com.yahoo.prelude.hitfield.RawData": {
     "superClass": "java.lang.Object",
     "interfaces": [],
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/Base64DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Base64DataField.java
new file mode 100644
index 00000000000..d51bdc0fad1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Base64DataField.java
@@ -0,0 +1,25 @@
+package com.yahoo.prelude.fastsearch;
+
+import com.yahoo.data.access.Inspector;
+import com.yahoo.data.access.simple.Value;
+import com.yahoo.prelude.hitfield.RawBase64;
+
+/**
+ * Represents a binary field that is presented as base64
+ * @author baldersheim
+ */
+public class Base64DataField extends DocsumField {
+    public Base64DataField(String name) {
+        super(name);
+    }
+
+    @Override
+    public String toString() {
+        return "field " + getName() + " type raw";
+    }
+
+    @Override
+    public Object convert(Inspector value) {
+        return new RawBase64(value.asData(Value.empty().asData()));
+    }
+}
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
index de07839e3e3..af7d98311f6 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
@@ -23,7 +23,7 @@ public class DataField extends DocsumField {
         super(name);
     }
 
-    private Object convert(byte[] value) {
+    private RawData convert(byte[] value) {
         return new RawData(value);
     }
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
index ef892585d21..70ffc71495a 100644
--- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
+++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
@@ -51,6 +51,7 @@ public abstract class DocsumField {
             fieldFactory.put("double", DoubleField.class);
             fieldFactory.put("string", StringField.class);
             fieldFactory.put("data", DataField.class);
+            fieldFactory.put("raw", Base64DataField.class);
             fieldFactory.put("longstring", LongstringField.class);
             fieldFactory.put("longdata", LongdataField.class);
             fieldFactory.put("jsonstring", StructDataField.class);
diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java
new file mode 100644
index 00000000000..134d0bc902a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java
@@ -0,0 +1,18 @@
+package com.yahoo.prelude.hitfield;
+
+import java.util.Base64;
+
+/**
+ * @author baldersheim
+ */
+public class RawBase64 {
+    private final byte[] content;
+    public RawBase64(byte[] content) {
+        this.content = content;
+    }
+
+    @Override
+    public String toString() {
+        return Base64.getEncoder().encodeToString(content);
+    }
+}