diff options
author | Jon Bratseth <bratseth@gmail.com> | 2021-09-19 22:03:39 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2021-09-19 22:03:39 +0200 |
commit | 03cca9c42b32d8d4ba24b348f8466de42bb05cd7 (patch) | |
tree | a6348759bf87ddb7cece3e74477067dc3981188a /docprocs/src | |
parent | 7ccaece3a2d065de9eb5a4cb18bc2f02d908d595 (diff) |
Add 'encode' expression
Diffstat (limited to 'docprocs/src')
3 files changed, 17 insertions, 10 deletions
diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java index 285f3e83602..53709c4ff87 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java @@ -20,6 +20,8 @@ import com.yahoo.document.DocumentUpdate; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.language.Linguistics; import java.util.logging.Level; + +import com.yahoo.language.process.Encoder; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.indexinglanguage.AdapterFactory; import com.yahoo.vespa.indexinglanguage.SimpleAdapterFactory; @@ -52,9 +54,10 @@ public class IndexingProcessor extends DocumentProcessor { @Inject public IndexingProcessor(DocumentmanagerConfig documentmanagerConfig, IlscriptsConfig ilscriptsConfig, - Linguistics linguistics) { + Linguistics linguistics, + Encoder encoder) { docTypeMgr = DocumentTypeManagerConfigurer.configureNewManager(documentmanagerConfig); - scriptMgr = new ScriptManager(docTypeMgr, ilscriptsConfig, linguistics); + scriptMgr = new ScriptManager(docTypeMgr, ilscriptsConfig, linguistics, encoder); adapterFactory = new SimpleAdapterFactory(new ExpressionSelector()); } diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java index a148966c250..8f3f75af795 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java @@ -5,6 +5,8 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.language.Linguistics; import java.util.logging.Level; + +import com.yahoo.language.process.Encoder; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; @@ -26,9 +28,9 @@ public class ScriptManager { private final Map<String, Map<String, DocumentScript>> documentFieldScripts; private final DocumentTypeManager docTypeMgr; - public ScriptManager(DocumentTypeManager docTypeMgr, IlscriptsConfig config, Linguistics linguistics) { + public ScriptManager(DocumentTypeManager docTypeMgr, IlscriptsConfig config, Linguistics linguistics, Encoder encoder) { this.docTypeMgr = docTypeMgr; - documentFieldScripts = createScriptsMap(docTypeMgr, config, linguistics); + documentFieldScripts = createScriptsMap(docTypeMgr, config, linguistics, encoder); } @@ -72,9 +74,10 @@ public class ScriptManager { private static Map<String, Map<String, DocumentScript>> createScriptsMap(DocumentTypeManager docTypeMgr, IlscriptsConfig config, - Linguistics linguistics) { + Linguistics linguistics, + Encoder encoder) { Map<String, Map<String, DocumentScript>> documentFieldScripts = new HashMap<>(config.ilscript().size()); - ScriptParserContext parserContext = new ScriptParserContext(linguistics); + ScriptParserContext parserContext = new ScriptParserContext(linguistics, encoder); parserContext.getAnnotatorConfig().setMaxTermOccurrences(config.maxtermoccurrences()); parserContext.getAnnotatorConfig().setMaxTokenLength(config.fieldmatchmaxlength()); diff --git a/docprocs/src/test/java/com/yahoo/docprocs/indexing/ScriptManagerTestCase.java b/docprocs/src/test/java/com/yahoo/docprocs/indexing/ScriptManagerTestCase.java index 23df8c0eb25..607fee4f10d 100644 --- a/docprocs/src/test/java/com/yahoo/docprocs/indexing/ScriptManagerTestCase.java +++ b/docprocs/src/test/java/com/yahoo/docprocs/indexing/ScriptManagerTestCase.java @@ -3,6 +3,7 @@ package com.yahoo.docprocs.indexing; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.language.process.Encoder; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.indexinglanguage.parser.ParseException; import org.junit.Test; @@ -28,7 +29,7 @@ public class ScriptManagerTestCase { IlscriptsConfig.Builder config = new IlscriptsConfig.Builder(); config.ilscript(new IlscriptsConfig.Ilscript.Builder().doctype("newssummary") .content("index")); - ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null); + ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null, Encoder.throwsOnUse); assertNotNull(scriptMgr.getScript(typeMgr.getDocumentType("newsarticle"))); assertNull(scriptMgr.getScript(new DocumentType("unknown"))); } @@ -43,7 +44,7 @@ public class ScriptManagerTestCase { IlscriptsConfig.Builder config = new IlscriptsConfig.Builder(); config.ilscript(new IlscriptsConfig.Ilscript.Builder().doctype("newsarticle") .content("index")); - ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null); + ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null, Encoder.throwsOnUse); assertNotNull(scriptMgr.getScript(typeMgr.getDocumentType("newssummary"))); assertNull(scriptMgr.getScript(new DocumentType("unknown"))); } @@ -52,7 +53,7 @@ public class ScriptManagerTestCase { public void requireThatEmptyConfigurationDoesNotThrow() { DocumentTypeManager typeMgr = new DocumentTypeManager(); typeMgr.configure("file:src/test/cfg/documentmanager_inherit.cfg"); - ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(new IlscriptsConfig.Builder()), null); + ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(new IlscriptsConfig.Builder()), null, Encoder.throwsOnUse); assertNull(scriptMgr.getScript(new DocumentType("unknown"))); } @@ -60,7 +61,7 @@ public class ScriptManagerTestCase { public void requireThatUnknownDocumentTypeReturnsNull() { DocumentTypeManager typeMgr = new DocumentTypeManager(); typeMgr.configure("file:src/test/cfg/documentmanager_inherit.cfg"); - ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(new IlscriptsConfig.Builder()), null); + ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(new IlscriptsConfig.Builder()), null, Encoder.throwsOnUse); for (Iterator<DocumentType> it = typeMgr.documentTypeIterator(); it.hasNext(); ) { assertNull(scriptMgr.getScript(it.next())); } |