diff options
author | Jon Bratseth <bratseth@gmail.com> | 2020-08-06 11:41:07 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2020-08-06 11:41:07 +0200 |
commit | 374d8876d0a877bbf954f9d29a15ca3151729028 (patch) | |
tree | 4a4fb314594ca126fac60ce8a0d26ee83f9498cd /config-model/src | |
parent | f450159eb48bd2dc2faed988a26ce1d4341539be (diff) |
Move any single set_language operation to the top
Diffstat (limited to 'config-model/src')
22 files changed, 226 insertions, 148 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 1fab30f9ea4..779ae54c242 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -396,23 +396,25 @@ public class SearchBuilder { return builder; } - public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException { - return createFromDirectory(dir, new RankProfileRegistry()); + public static SearchBuilder createFromDirectory(String dir, DeployLogger logger) throws IOException, ParseException { + return createFromDirectory(dir, new RankProfileRegistry(), logger); } public static SearchBuilder createFromDirectory(String dir, - RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { - return createFromDirectory(dir, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir)); + RankProfileRegistry rankProfileRegistry, + DeployLogger logger) throws IOException, ParseException { + return createFromDirectory(dir, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir), logger); } public static SearchBuilder createFromDirectory(String dir, RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + QueryProfileRegistry queryProfileRegistry, + DeployLogger logger) throws IOException, ParseException { SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), rankProfileRegistry, queryProfileRegistry); for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { builder.importFile(i.next()); } - builder.build(true, new BaseDeployLogger()); + builder.build(true, logger); return builder; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java index 7cfcf422e80..d6065a07656 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java @@ -39,14 +39,11 @@ public abstract class Derived implements Exportable { protected void derive(Search search) { setName(search.getName()); derive(search.getDocument(), search); - for (Index index : search.getExplicitIndices()) { + for (Index index : search.getExplicitIndices()) derive(index, search); - } - for (SDField field : search.allExtraFields() ) { + for (SDField field : search.allExtraFields()) derive(field, search); - } - search.allImportedFields() - .forEach(importedField -> derive(importedField, search)); + search.allImportedFields().forEach(importedField -> derive(importedField, search)); } @@ -54,10 +51,10 @@ public abstract class Derived implements Exportable { * Derives the content of this configuration. This * default calls derive(SDField) for each document field */ - protected void derive(SDDocumentType document,Search search) { + protected void derive(SDDocumentType document, Search search) { for (Field field : document.fieldSet()) { SDField sdField = (SDField) field; - if (!sdField.isExtraField()) { + if ( ! sdField.isExtraField()) { derive(sdField, search); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java index 94d475cf519..8a12bb92dae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.PassthroughExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; @@ -24,6 +25,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * An indexing language script derived from a search definition. An indexing script contains a set of indexing @@ -35,30 +37,49 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro private final List<String> docFields = new ArrayList<>(); private final List<Expression> expressions = new ArrayList<>(); + private List<ImmutableSDField> fieldsSettingLanguage; public IndexingScript(Search search) { derive(search); } @Override + protected void derive(Search search) { + fieldsSettingLanguage = fieldsSettingLanguage(search); + if (fieldsSettingLanguage.size() == 1) // Assume this language should be used for all fields + addExpression(fieldsSettingLanguage.get(0).getIndexingScript()); + super.derive(search); + } + + @Override protected void derive(ImmutableSDField field, Search search) { - if (field.isImportedField()) { - return; - } - if (field.hasFullIndexingDocprocRights()) { + if (field.isImportedField()) return; + + if (field.hasFullIndexingDocprocRights()) docFields.add(field.getName()); - } + if (field.usesStructOrMap() && - !field.getDataType().equals(PositionDataType.INSTANCE) && - !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) - { + ! field.getDataType().equals(PositionDataType.INSTANCE) && + ! field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { return; // unsupported } - ScriptExpression script = field.getIndexingScript(); - if (!script.isEmpty()) { - expressions.add(new StatementExpression(new ClearStateExpression(), - new GuardExpression(script))); - } + + if (fieldsSettingLanguage.size() == 1 && fieldsSettingLanguage.get(0).equals(field)) + return; // Already added + + addExpression(field.getIndexingScript()); + } + + private void addExpression(ScriptExpression expression) { + if ( expression.isEmpty()) return; + expressions.add(new StatementExpression(new ClearStateExpression(), new GuardExpression(expression))); + } + + private List<ImmutableSDField> fieldsSettingLanguage(Search search) { + return search.allFieldsList().stream() + .filter(field -> ! field.isImportedField()) + .filter(field -> field.containsExpression(SetLanguageExpression.class)) + .collect(Collectors.toList()); } public Iterable<Expression> expressions() { @@ -81,20 +102,19 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) { ArrayList<Expression> later = new ArrayList<>(); - Set<String> touchedFields = new HashSet<String>(); - for (Expression exp : expressions) { + Set<String> touchedFields = new HashSet<>(); + for (Expression expression : expressions) { + if (modifiesSelf(expression) && ! setsLanguage(expression)) + later.add(expression); + else + ilscriptBuilder.content(expression.toString()); + FieldScanVisitor fieldFetcher = new FieldScanVisitor(); - if (modifiesSelf(exp)) { - later.add(exp); - } else { - ilscriptBuilder.content(exp.toString()); - } - fieldFetcher.visit(exp); + fieldFetcher.visit(expression); touchedFields.addAll(fieldFetcher.touchedFields()); } - for (Expression exp : later) { + for (Expression exp : later) ilscriptBuilder.content(exp.toString()); - } generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields); } @@ -110,13 +130,20 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro } } - private boolean modifiesSelf(Expression exp) { - MyExpVisitor visitor = new MyExpVisitor(); - visitor.visit(exp); + private boolean setsLanguage(Expression expression) { + SetsLanguageVisitor visitor = new SetsLanguageVisitor(); + visitor.visit(expression); + return visitor.setsLanguage; + } + + private boolean modifiesSelf(Expression expression) { + ModifiesSelfVisitor visitor = new ModifiesSelfVisitor(); + visitor.visit(expression); return visitor.modifiesSelf(); } - private class MyExpVisitor extends ExpressionVisitor { + private static class ModifiesSelfVisitor extends ExpressionVisitor { + private String inputField = null; private String outputField = null; @@ -124,9 +151,8 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro @Override protected void doVisit(Expression expression) { - if (modifiesSelf()) { - return; - } + if (modifiesSelf()) return; + if (expression instanceof InputExpression) { inputField = ((InputExpression) expression).getFieldName(); } @@ -136,6 +162,18 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro } } + private static class SetsLanguageVisitor extends ExpressionVisitor { + + boolean setsLanguage = false; + + @Override + protected void doVisit(Expression expression) { + if (expression instanceof SetLanguageExpression) + setsLanguage = true; + } + + } + private static class FieldScanVisitor extends ExpressionVisitor { List<String> touchedFields = new ArrayList<String>(); List<String> candidates = new ArrayList<String>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index e36635ba6b8..c3a0abc892c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -465,9 +465,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } @Override - public ScriptExpression getIndexingScript() { - return indexingScript; - } + public ScriptExpression getIndexingScript() { return indexingScript; } @SuppressWarnings("deprecation") @Override @@ -477,7 +475,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, stemming=Stemming.NONE; } this.dataType = type; - if (!idOverride) { + if ( ! idOverride) { this.fieldId = calculateIdV7(null); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java index 8f2a29abcb6..416d956ac21 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java @@ -29,7 +29,7 @@ public class OptimizeIlscript extends Processor { field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script)); if ( ! field.getIndexingScript().toString().equals(script.toString())) { - warn(search, field, "Rewrote ilscript from:\n" + script.toString() + + info(search, field, "Rewrote ilscript from:\n" + script.toString() + "\nto\n" + field.getIndexingScript().toString()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 3f225b00277..e8594c2a87f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -27,7 +27,6 @@ public class Processing { IndexFieldNames::new, IntegerIndex2Attribute::new, MakeAliases::new, - SetLanguage::new, UriHack::new, LiteralBoost::new, TagType::new, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index e15e17817a2..3744af7cc2c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -128,12 +128,22 @@ public abstract class Processor { throw newProcessException(search, field, msg); } - protected void warn(String searchName, String fieldName, String msg) { - String fullMsg = formatError(searchName, fieldName, msg); + protected void warn(String searchName, String fieldName, String message) { + String fullMsg = formatError(searchName, fieldName, message); deployLogger.log(Level.WARNING, fullMsg); } - protected void warn(Search search, Field field, String msg) { - warn(search.getName(), field.getName(), msg); + protected void warn(Search search, Field field, String message) { + warn(search.getName(), field.getName(), message); } + + protected void info(String searchName, String fieldName, String message) { + String fullMsg = formatError(searchName, fieldName, message); + deployLogger.log(Level.INFO, fullMsg); + } + + protected void info(Search search, Field field, String message) { + warn(search.getName(), field.getName(), message); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java deleted file mode 100644 index 8a4795c4dd2..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Search; -import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.ArrayList; -import java.util.List; - -/** - * Check that no text field appears before a field that sets language. - * - * @author Gunnar Gauslaa Bergem - */ -public class SetLanguage extends Processor { - - public SetLanguage(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(search, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - List<String> textFieldsWithoutLanguage = new ArrayList<>(); - - for (SDField field : search.allConcreteFields()) { - if (fieldMustComeAfterLanguageSettingField(field)) { - textFieldsWithoutLanguage.add(field.getName()); - } - if (field.containsExpression(SetLanguageExpression.class) && !textFieldsWithoutLanguage.isEmpty()) { - StringBuilder fieldString = new StringBuilder(); - for (String fieldName : textFieldsWithoutLanguage) { - fieldString.append(fieldName).append(" "); - } - warn(search, field, "Field '" + field.getName() + "' sets the language for this document, " + - "and should be defined as the first field in the searchdefinition." + - "Preceding text fields that will not have their language set: " + - fieldString.toString() + - " (This warning is omitted for any subsequent fields that also do set_language.)"); - return; - } - } - } - - private boolean fieldMustComeAfterLanguageSettingField(SDField field) { - return (!field.containsExpression(SetLanguageExpression.class) && - (field.getDataType() == DataType.STRING)); - } - -} diff --git a/config-model/src/test/derived/language/ilscripts.cfg b/config-model/src/test/derived/language/ilscripts.cfg new file mode 100644 index 00000000000..cafc7feafb8 --- /dev/null +++ b/config-model/src/test/derived/language/ilscripts.cfg @@ -0,0 +1,9 @@ +maxtermoccurrences 100 +fieldmatchmaxlength 1000000 +ilscript[].doctype "language" +ilscript[].docfield[] "language" +ilscript[].docfield[] "title" +ilscript[].content[] "clear_state | guard { input language | tokenize normalize stem:\"BEST\" | summary language | index language | set_language; }" +ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | index titlebest; }" +ilscript[].content[] "clear_state | guard { input title | tokenize normalize | index titlenone; }" +ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }"
\ No newline at end of file diff --git a/config-model/src/test/derived/language/language.sd b/config-model/src/test/derived/language/language.sd new file mode 100644 index 00000000000..a859438d885 --- /dev/null +++ b/config-model/src/test/derived/language/language.sd @@ -0,0 +1,19 @@ +schema language { + document language { + field language type string { + indexing: summary | index |set_language + } + field title type string { + indexing: summary | index + } + } + field titlenone type string { + indexing: input title | index + stemming: none + } + field titlebest type string { + indexing: input title | index + stemming: best + } + +}
\ No newline at end of file diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg index 3bcdd16e3d6..b3da5f8e727 100644 --- a/config-model/src/test/derived/types/ilscripts.cfg +++ b/config-model/src/test/derived/types/ilscripts.cfg @@ -21,6 +21,7 @@ ilscript[].docfield[] "doublemapfield" ilscript[].docfield[] "arraymapfield" ilscript[].docfield[] "arrarr" ilscript[].docfield[] "maparr" +ilscript[].docfield[] "complexarray" ilscript[].docfield[] "mystructfield" ilscript[].docfield[] "mystructmap" ilscript[].docfield[] "mystructarr" @@ -28,7 +29,6 @@ ilscript[].docfield[] "Folders" ilscript[].docfield[] "juletre" ilscript[].docfield[] "album0" ilscript[].docfield[] "album1" -ilscript[].docfield[] "complexarray" ilscript[].content[] "clear_state | guard { input along | attribute other; }" ilscript[].content[] "clear_state | guard { input abyte | summary abyte | attribute abyte; }" ilscript[].content[] "clear_state | guard { input along | summary along | attribute along; }" diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java index 7f3ea7d14bc..2e0fc3b4f98 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java @@ -16,9 +16,12 @@ public abstract class SchemaTestCase { assertSerializedConfigFileEquals(filename, cfg); } - protected static void assertConfigFiles(String expectedFile, String cfgFile, boolean updateOnAssert) throws IOException { + protected static void assertConfigFiles(String expectedFile, + String cfgFile, + boolean orderMatters, + boolean updateOnAssert) throws IOException { try { - assertSerializedConfigEquals(readAndCensorIndexes(expectedFile), readAndCensorIndexes(cfgFile)); + assertSerializedConfigEquals(readAndCensorIndexes(expectedFile), readAndCensorIndexes(cfgFile), orderMatters); } catch (AssertionError e) { if (updateOnAssert) { BufferedWriter writer = IOUtils.createWriter(expectedFile, false); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java index a345cabe909..6b6af95a8d2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; @@ -26,21 +26,25 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase { private static final String tempDir = "temp/"; private static final String searchDefRoot = "src/test/derived/"; - private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException { + private DerivedConfiguration derive(String dirName, + String searchDefinitionName, + TestProperties properties, + DeployLogger logger) throws IOException, ParseException { File toDir = new File(tempDir + dirName); toDir.mkdirs(); deleteContent(toDir); - SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/"); - return derive(dirName, searchDefinitionName, properties, builder); + SearchBuilder builder = SearchBuilder.createFromDirectory(searchDefRoot + dirName + "/", logger); + return derive(dirName, searchDefinitionName, properties, builder, logger); } private DerivedConfiguration derive(String dirName, String searchDefinitionName, TestProperties properties, - SearchBuilder builder) throws IOException { + SearchBuilder builder, + DeployLogger logger) throws IOException { DerivedConfiguration config = new DerivedConfiguration(builder.getSearch(searchDefinitionName), - new BaseDeployLogger(), + logger, properties, builder.getRankProfileRegistry(), builder.getQueryProfileRegistry(), @@ -80,15 +84,23 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase { * @throws IOException if file access failed. */ protected DerivedConfiguration assertCorrectDeriving(String dirName) throws IOException, ParseException { - return assertCorrectDeriving(dirName, null); + return assertCorrectDeriving(dirName, new TestableDeployLogger()); + } + protected DerivedConfiguration assertCorrectDeriving(String dirName, DeployLogger logger) throws IOException, ParseException { + return assertCorrectDeriving(dirName, null, logger); } - protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName) throws IOException, ParseException { - return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties()); + protected DerivedConfiguration assertCorrectDeriving(String dirName, + String searchDefinitionName, + DeployLogger logger) throws IOException, ParseException { + return assertCorrectDeriving(dirName, searchDefinitionName, new TestProperties(), logger); } - protected DerivedConfiguration assertCorrectDeriving(String dirName, String searchDefinitionName, TestProperties properties) throws IOException, ParseException { - DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties); + protected DerivedConfiguration assertCorrectDeriving(String dirName, + String searchDefinitionName, + TestProperties properties, + DeployLogger logger) throws IOException, ParseException { + DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties, logger); assertCorrectConfigFiles(dirName); return derived; } @@ -97,9 +109,9 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase { * Asserts config is correctly derived given a builder. * This will fail if the builder contains multiple search definitions. */ - protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName) throws IOException { + protected DerivedConfiguration assertCorrectDeriving(SearchBuilder builder, String dirName, DeployLogger logger) throws IOException { builder.build(); - DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder); + DerivedConfiguration derived = derive(dirName, null, new TestProperties(), builder, logger); assertCorrectConfigFiles(dirName); return derived; } @@ -121,13 +133,14 @@ public abstract class AbstractExportingTestCase extends SchemaTestCase { if (files == null) return; for (File file : files) { if ( ! file.getName().endsWith(".cfg")) continue; - assertEqualFiles(file.getPath(), tempDir + name + "/" + file.getName()); + boolean orderMatters = file.getName().equals("ilscripts.cfg"); + assertEqualFiles(file.getPath(), tempDir + name + "/" + file.getName(), orderMatters); } } - static void assertEqualFiles(String correctFileName, String checkFileName) throws IOException { + static void assertEqualFiles(String correctFileName, String checkFileName, boolean orderMatters) throws IOException { // Set updateOnAssert to true if you want update the files with correct answer. - assertConfigFiles(correctFileName, checkFileName, false); + assertConfigFiles(correctFileName, checkFileName, orderMatters, false); } void deleteContent(File dir) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index 71cca20f8fa..770bea55af2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -1,13 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.searchdefinition.SearchBuilder; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; import java.io.IOException; +import static org.junit.Assert.assertEquals; + /** * Tests exporting * @@ -158,7 +159,14 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testRankProfileInheritance() throws IOException, ParseException { - assertCorrectDeriving("rankprofileinheritance", "child"); + assertCorrectDeriving("rankprofileinheritance", "child", new TestableDeployLogger()); + } + + @Test + public void testLanguage() throws IOException, ParseException { + TestableDeployLogger logger = new TestableDeployLogger(); + assertCorrectDeriving("language", logger); + assertEquals(0, logger.warnings.size()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java index a0dd89229dd..f61143833c9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java @@ -13,26 +13,26 @@ public class ImportedFieldsTestCase extends AbstractExportingTestCase { @Test public void configs_for_imported_fields_are_derived() throws IOException, ParseException { - assertCorrectDeriving("importedfields", "child"); + assertCorrectDeriving("importedfields", "child", new TestableDeployLogger()); } @Test public void configs_for_imported_struct_fields_are_derived() throws IOException, ParseException { - assertCorrectDeriving("imported_struct_fields", "child"); + assertCorrectDeriving("imported_struct_fields", "child", new TestableDeployLogger()); } @Test public void configs_for_imported_position_field_are_derived() throws IOException, ParseException { - assertCorrectDeriving("imported_position_field", "child"); + assertCorrectDeriving("imported_position_field", "child", new TestableDeployLogger()); } @Test public void configs_for_imported_position_field_summary_are_derived() throws IOException, ParseException { - assertCorrectDeriving("imported_position_field_summary", "child"); + assertCorrectDeriving("imported_position_field_summary", "child", new TestableDeployLogger()); } @Test public void derives_configs_for_imported_fields_when_reference_fields_are_inherited() throws IOException, ParseException { - assertCorrectDeriving("imported_fields_inherited_reference", "child_c"); + assertCorrectDeriving("imported_fields_inherited_reference", "child_c", new TestableDeployLogger()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java index 187e766c315..5e45102a626 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java @@ -18,7 +18,7 @@ public class MailTestCase extends AbstractExportingTestCase { String dir = "src/test/derived/mail/"; SearchBuilder sb = new SearchBuilder(); sb.importFile(dir + "mail.sd"); - assertCorrectDeriving(sb, dir); + assertCorrectDeriving(sb, dir, new TestableDeployLogger()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java index 073a15b0525..1f1c905e0bf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java @@ -13,6 +13,6 @@ public class ReferenceFieldsTestCase extends AbstractExportingTestCase { @Test public void configs_related_to_reference_fields_are_derived() throws IOException, ParseException { - assertCorrectDeriving("reference_fields", "ad"); + assertCorrectDeriving("reference_fields", "ad", new TestableDeployLogger()); } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java index 45a6240d3ba..83e11c365f8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java @@ -45,10 +45,8 @@ public class SimpleInheritTestCase extends AbstractExportingTestCase { private void checkDir(String toDirName, String expectedResultsDirName) throws IOException { File[] files = new File(expectedResultsDirName).listFiles(); for (File file : files) { - if (!file.getName().endsWith(".cfg")) { - continue; - } - assertEqualFiles(file.getPath(), toDirName + "/" + file.getName()); + if ( ! file.getName().endsWith(".cfg")) continue; + assertEqualFiles(file.getPath(), toDirName + "/" + file.getName(), false); } } } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java new file mode 100644 index 00000000000..a23c8024872 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.application.api.DeployLogger; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author bratseth + */ +public class TestableDeployLogger implements DeployLogger { + + private static final Logger log = Logger.getLogger("DeployLogger"); + + public List<String> warnings = new ArrayList<>(); + public List<String> info = new ArrayList<>(); + + @Override + public final void log(Level level, String message) { + log.log(level, message); + if (level.equals(Level.WARNING)) + warnings.add(message); + if (level.equals(Level.INFO)) + info.add(message); + } + +} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index 1567a4c3b5e..8861432d97b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -9,6 +9,7 @@ import com.yahoo.searchdefinition.*; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.searchdefinition.derived.AttributeFields; import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.derived.TestableDeployLogger; import com.yahoo.searchdefinition.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; @@ -25,7 +26,8 @@ public class RankingExpressionsTestCase extends SchemaTestCase { public void testFunctions() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressionfunction", - rankProfileRegistry).getSearch(); + rankProfileRegistry, + new TestableDeployLogger()).getSearch(); RankProfile functionsRankProfile = rankProfileRegistry.get(search, "macros"); Map<String, RankProfile.RankingExpressionFunction> functions = functionsRankProfile.getFunctions(); assertEquals(2, functions.get("titlematch$").function().arguments().size()); @@ -63,7 +65,9 @@ public class RankingExpressionsTestCase extends SchemaTestCase { @Test(expected = IllegalArgumentException.class) public void testThatIncludingFileInSubdirFails() throws IOException, ParseException { RankProfileRegistry registry = new RankProfileRegistry(); - Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressioninfile", registry).getSearch(); + Search search = SearchBuilder.createFromDirectory("src/test/examples/rankingexpressioninfile", + registry, + new TestableDeployLogger()).getSearch(); new DerivedConfiguration(search, new BaseDeployLogger(), new TestProperties(), registry, new QueryProfileRegistry(), new ImportedMlModels()); // rank profile parsing happens during deriving } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java index 31d0084570a..9e7370a933c 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java @@ -159,7 +159,7 @@ public class RoutingTestCase { fail("Expected file '" + fileName + "' not found."); return; } - assertSerializedConfigEquals(IOUtils.readFile(files.get(fileName)), configString); + assertSerializedConfigEquals(IOUtils.readFile(files.get(fileName)), configString, false); } /** diff --git a/config-model/src/test/java/helpers/CompareConfigTestHelper.java b/config-model/src/test/java/helpers/CompareConfigTestHelper.java index 89b4eeb3c8d..586f82ec379 100644 --- a/config-model/src/test/java/helpers/CompareConfigTestHelper.java +++ b/config-model/src/test/java/helpers/CompareConfigTestHelper.java @@ -19,15 +19,20 @@ import static org.junit.Assert.assertEquals; public class CompareConfigTestHelper { public static void assertSerializedConfigFileEquals(String filename, String actual) throws IOException { - assertSerializedConfigEquals(IOUtils.readFile(new File(filename)), actual); + assertSerializedConfigEquals(IOUtils.readFile(new File(filename)), actual, false); } // Written this way to compare order independently but output error with order preserved // Note that this means that if a test fails you'll also see spurious differences in the comparison // from lines which are present in both but at different locations. - public static void assertSerializedConfigEquals(String expected, String actual) { - if ( ! sortLines(expected.trim()).equals(sortLines(actual.trim()))) - assertEquals(expected, actual); + public static void assertSerializedConfigEquals(String expected, String actual, boolean orderMatters) { + if (orderMatters) { + assertEquals(expected.trim(), actual.trim()); + } + else { + if (!sortLines(expected.trim()).equals(sortLines(actual.trim()))) + assertEquals(expected, actual); + } } private static String sortLines(String fileData) { |