diff options
Diffstat (limited to 'config-model/src')
3 files changed, 51 insertions, 0 deletions
diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg new file mode 100644 index 00000000000..ead74110db3 --- /dev/null +++ b/config-model/src/test/derived/tokenization/ilscripts.cfg @@ -0,0 +1,9 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "tokenization" +ilscript[].docfield[] "text" +ilscript[].docfield[] "text_array" +ilscript[].content[] "clear_state | guard { input text_array | for_each { lowercase } | for_each { normalize } | for_each { tokenize normalize stem:\"BEST\" } | index text_array_derived | summary text_array_derived; }" +ilscript[].content[] "clear_state | guard { input text | normalize | tokenize normalize stem:\"BEST\" | index text_derived | summary text_derived; }" +ilscript[].content[] "clear_state | guard { input text | tokenize normalize stem:\"BEST\" | index text | summary text; }" +ilscript[].content[] "clear_state | guard { input text_array | for_each { tokenize normalize stem:\"BEST\" } | index text_array | summary text_array; }"
\ No newline at end of file diff --git a/config-model/src/test/derived/tokenization/tokenization.sd b/config-model/src/test/derived/tokenization/tokenization.sd new file mode 100644 index 00000000000..4510a574d60 --- /dev/null +++ b/config-model/src/test/derived/tokenization/tokenization.sd @@ -0,0 +1,23 @@ +schema tokenization { + + document tokenization { + + field text type string { + indexing: index | summary + } + + field text_array type array<string> { + indexing: index | summary + } + + } + + field text_derived type string { + indexing: input text | normalize | index | summary + } + + field text_array_derived type array<string> { + indexing: input text_array | for_each { lowercase } | for_each { normalize } | index | summary + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java new file mode 100755 index 00000000000..6fe367ef6d1 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java @@ -0,0 +1,19 @@ +// Copyright Verizon media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author bratseh + */ +public class TokenizationTestCase extends AbstractExportingTestCase { + + @Test + public void testTokenizationScripts() throws IOException, ParseException { + assertCorrectDeriving("tokenization"); + } + +} |