diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-05-19 21:51:00 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2021-05-19 21:51:00 +0200 |
commit | d468c1aff08b6d13763df9bbdfda958ec723bb75 (patch) | |
tree | fae114bfbf33773e5012523abbecd9c3501b06d2 | |
parent | 2dd947c284dafb307522747c0b9c07092fabdca3 (diff) |
Prepare for raw fields to be presented as base64 encoded in summary.
10 files changed, 117 insertions, 7 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index 64c5590b689..9b7434dccab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -61,6 +61,8 @@ public class Search implements ImmutableSearch { /** True if this doesn't define a search, just a document type */ private boolean documentsOnly = false; + private boolean rawAsBase64 = false; + /** The stemming setting of this search definition. Default is BEST. */ private Stemming stemming = Stemming.BEST; @@ -125,6 +127,16 @@ public class Search implements ImmutableSearch { } /** + * Returns true if 'raw' fields shall be presented as base64 in summary + * Note that tis is temporary and will disappear on Vespa 8 as it will become default, and only option. + * + * @return true if raw shall be encoded as base64 in summary + */ + public boolean isRawAsBase64() { return rawAsBase64; } + + public void enableRawAsBase64() { rawAsBase64 = true; } + + /** * Sets the stemming default of fields. Default is ALL * * @param stemming set default stemming for this searchdefinition diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java index b08e9948ecd..af2168545dc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -27,6 +27,7 @@ public class SummaryClass extends Derived { /** True if this summary class needs to access summary information on disk */ private boolean accessingDiskSummary = false; + private final boolean rawAsBase64; /** The summary fields of this indexed by name */ private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); @@ -42,6 +43,7 @@ public class SummaryClass extends Derived { */ public SummaryClass(Search search, DocumentSummary summary, DeployLogger deployLogger) { this.deployLogger = deployLogger; + this.rawAsBase64 = search.isRawAsBase64(); deriveName(summary); deriveFields(search,summary); deriveImplicitFields(summary); @@ -74,12 +76,12 @@ public class SummaryClass extends Derived { private void addField(String name, DataType type, SummaryTransform transform) { if (fields.containsKey(name)) { SummaryClassField sf = fields.get(name); - if (!SummaryClassField.convertDataType(type, transform).equals(sf.getType())) { + if (!SummaryClassField.convertDataType(type, transform, rawAsBase64).equals(sf.getType())) { deployLogger.logApplicationPackage(Level.WARNING, "Conflicting definition of field " + name + ". " + "Declared as type " + sf.getType() + " and " + type); } } else { - fields.put(name, new SummaryClassField(name, type, transform)); + fields.put(name, new SummaryClassField(name, type, transform, rawAsBase64)); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java index c0093e031d1..3c29971a74c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java @@ -44,6 +44,7 @@ public class SummaryClassField { DOUBLE("double"), STRING("string"), DATA("data"), + RAW("raw"), LONGSTRING("longstring"), LONGDATA("longdata"), XMLSTRING("xmlstring"), @@ -67,9 +68,9 @@ public class SummaryClassField { } } - public SummaryClassField(String name, DataType type, SummaryTransform transform) { + public SummaryClassField(String name, DataType type, SummaryTransform transform, boolean rawAsBase64) { this.name = name; - this.type = convertDataType(type, transform); + this.type = convertDataType(type, transform, rawAsBase64); } public String getName() { return name; } @@ -77,7 +78,7 @@ public class SummaryClassField { public Type getType() { return type; } /** Converts to the right summary field type from a field datatype and a transform*/ - public static Type convertDataType(DataType fieldType, SummaryTransform transform) { + public static Type convertDataType(DataType fieldType, SummaryTransform transform, boolean rawAsBase64) { FieldValue fval = fieldType.createFieldValue(); if (fval instanceof StringFieldValue) { if (transform != null && transform.equals(SummaryTransform.RANKFEATURES)) { @@ -102,7 +103,7 @@ public class SummaryClassField { } else if (fval instanceof ByteFieldValue) { return Type.BYTE; } else if (fval instanceof Raw) { - return Type.DATA; + return rawAsBase64 ? Type.RAW : Type.DATA; } else if (fval instanceof Struct) { return Type.JSONSTRING; } else if (fval instanceof PredicateFieldValue) { diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 88c45a18e4e..8f99c388fb1 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -237,6 +237,7 @@ TOKEN : | < RANKPROFILE: "rank-profile" > | < RANKDEGRADATIONFREQ: "rank-degradation-frequency" > | < RANKDEGRADATION: "rank-degradation" > +| < RAW_AS_BASE64_IN_SUMMARY: "raw-as-base64-in-summary" > | < RPBINSIZE: "doc-frequency" > | < RPBINLOW: "min-fullrank-docs"> | < RPPOSBINSIZE: "occurrences-per-doc" > @@ -451,6 +452,7 @@ Search rootSchema(String dir) : Object rootSchemaItem(Search search) : { } { ( document(search) + | rawAsBase64(search) | documentSummary(search) | field(null, search) | index(search, null) @@ -561,6 +563,12 @@ Object documentBody(SDDocumentType document, Search search) : { return null; } } +void rawAsBase64(Search search) : +{} +{ + <RAW_AS_BASE64_IN_SUMMARY> { search.enableRawAsBase64(); } +} + /** * Consumes a document head block. * diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java index afbc9f52f6b..bfc738a4f87 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java @@ -23,6 +23,37 @@ import static org.junit.Assert.assertNull; public class SummaryTestCase extends SchemaTestCase { @Test + public void deriveRawAsBase64() throws ParseException { + String sd = joinLines( + "schema s {", + " raw-as-base64-in-summary", + " document s {", + " field raw_field type raw {", + " indexing: summary", + " }", + " }", + "}"); + Search search = SearchBuilder.createFromString(sd).getSearch(); + SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger()); + assertEquals(SummaryClassField.Type.RAW, summary.getField("raw_field").getType()); + } + + @Test + public void deriveRawAsLegacy() throws ParseException { + String sd = joinLines( + "schema s {", + " document s {", + " field raw_field type raw {", + " indexing: summary", + " }", + " }", + "}"); + Search search = SearchBuilder.createFromString(sd).getSearch(); + SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger()); + assertEquals(SummaryClassField.Type.DATA, summary.getField("raw_field").getType()); + } + + @Test public void testDeriving() throws IOException, ParseException { Search search = SearchBuilder.buildFromFile("src/test/examples/simple.sd"); SummaryClass summary = new SummaryClass(search, search.getSummary("default"), new BaseDeployLogger()); diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 74ed9d33f04..b577660c1b9 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -201,6 +201,18 @@ ], "fields": [] }, + "com.yahoo.prelude.hitfield.RawBase64": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(byte[])", + "public java.lang.String toString()" + ], + "fields": [] + }, "com.yahoo.prelude.hitfield.RawData": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/Base64DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Base64DataField.java new file mode 100644 index 00000000000..d51bdc0fad1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/Base64DataField.java @@ -0,0 +1,25 @@ +package com.yahoo.prelude.fastsearch; + +import com.yahoo.data.access.Inspector; +import com.yahoo.data.access.simple.Value; +import com.yahoo.prelude.hitfield.RawBase64; + +/** + * Represents a binary field that is presented as base64 + * @author baldersheim + */ +public class Base64DataField extends DocsumField { + public Base64DataField(String name) { + super(name); + } + + @Override + public String toString() { + return "field " + getName() + " type raw"; + } + + @Override + public Object convert(Inspector value) { + return new RawBase64(value.asData(Value.empty().asData())); + } +} diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java index de07839e3e3..af7d98311f6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java @@ -23,7 +23,7 @@ public class DataField extends DocsumField { super(name); } - private Object convert(byte[] value) { + private RawData convert(byte[] value) { return new RawData(value); } diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index ef892585d21..70ffc71495a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -51,6 +51,7 @@ public abstract class DocsumField { fieldFactory.put("double", DoubleField.class); fieldFactory.put("string", StringField.class); fieldFactory.put("data", DataField.class); + fieldFactory.put("raw", Base64DataField.class); fieldFactory.put("longstring", LongstringField.class); fieldFactory.put("longdata", LongdataField.class); fieldFactory.put("jsonstring", StructDataField.class); diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java new file mode 100644 index 00000000000..134d0bc902a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java @@ -0,0 +1,18 @@ +package com.yahoo.prelude.hitfield; + +import java.util.Base64; + +/** + * @author baldersheim + */ +public class RawBase64 { + private final byte[] content; + public RawBase64(byte[] content) { + this.content = content; + } + + @Override + public String toString() { + return Base64.getEncoder().encodeToString(content); + } +} |