diff options
Diffstat (limited to 'config-model/src/main/java/com')
8 files changed, 98 insertions, 109 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java index 1fab30f9ea4..779ae54c242 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -396,23 +396,25 @@ public class SearchBuilder { return builder; } - public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException { - return createFromDirectory(dir, new RankProfileRegistry()); + public static SearchBuilder createFromDirectory(String dir, DeployLogger logger) throws IOException, ParseException { + return createFromDirectory(dir, new RankProfileRegistry(), logger); } public static SearchBuilder createFromDirectory(String dir, - RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { - return createFromDirectory(dir, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir)); + RankProfileRegistry rankProfileRegistry, + DeployLogger logger) throws IOException, ParseException { + return createFromDirectory(dir, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir), logger); } public static SearchBuilder createFromDirectory(String dir, RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + QueryProfileRegistry queryProfileRegistry, + DeployLogger logger) throws IOException, ParseException { SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), rankProfileRegistry, queryProfileRegistry); for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { builder.importFile(i.next()); } - builder.build(true, new BaseDeployLogger()); + builder.build(true, logger); return builder; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java index 7cfcf422e80..d6065a07656 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java @@ -39,14 +39,11 @@ public abstract class Derived implements Exportable { protected void derive(Search search) { setName(search.getName()); derive(search.getDocument(), search); - for (Index index : search.getExplicitIndices()) { + for (Index index : search.getExplicitIndices()) derive(index, search); - } - for (SDField field : search.allExtraFields() ) { + for (SDField field : search.allExtraFields()) derive(field, search); - } - search.allImportedFields() - .forEach(importedField -> derive(importedField, search)); + search.allImportedFields().forEach(importedField -> derive(importedField, search)); } @@ -54,10 +51,10 @@ public abstract class Derived implements Exportable { * Derives the content of this configuration. This * default calls derive(SDField) for each document field */ - protected void derive(SDDocumentType document,Search search) { + protected void derive(SDDocumentType document, Search search) { for (Field field : document.fieldSet()) { SDField sdField = (SDField) field; - if (!sdField.isExtraField()) { + if ( ! sdField.isExtraField()) { derive(sdField, search); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java index 94d475cf519..8a12bb92dae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.PassthroughExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; @@ -24,6 +25,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * An indexing language script derived from a search definition. An indexing script contains a set of indexing @@ -35,30 +37,49 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro private final List<String> docFields = new ArrayList<>(); private final List<Expression> expressions = new ArrayList<>(); + private List<ImmutableSDField> fieldsSettingLanguage; public IndexingScript(Search search) { derive(search); } @Override + protected void derive(Search search) { + fieldsSettingLanguage = fieldsSettingLanguage(search); + if (fieldsSettingLanguage.size() == 1) // Assume this language should be used for all fields + addExpression(fieldsSettingLanguage.get(0).getIndexingScript()); + super.derive(search); + } + + @Override protected void derive(ImmutableSDField field, Search search) { - if (field.isImportedField()) { - return; - } - if (field.hasFullIndexingDocprocRights()) { + if (field.isImportedField()) return; + + if (field.hasFullIndexingDocprocRights()) docFields.add(field.getName()); - } + if (field.usesStructOrMap() && - !field.getDataType().equals(PositionDataType.INSTANCE) && - !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) - { + ! field.getDataType().equals(PositionDataType.INSTANCE) && + ! field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { return; // unsupported } - ScriptExpression script = field.getIndexingScript(); - if (!script.isEmpty()) { - expressions.add(new StatementExpression(new ClearStateExpression(), - new GuardExpression(script))); - } + + if (fieldsSettingLanguage.size() == 1 && fieldsSettingLanguage.get(0).equals(field)) + return; // Already added + + addExpression(field.getIndexingScript()); + } + + private void addExpression(ScriptExpression expression) { + if ( expression.isEmpty()) return; + expressions.add(new StatementExpression(new ClearStateExpression(), new GuardExpression(expression))); + } + + private List<ImmutableSDField> fieldsSettingLanguage(Search search) { + return search.allFieldsList().stream() + .filter(field -> ! field.isImportedField()) + .filter(field -> field.containsExpression(SetLanguageExpression.class)) + .collect(Collectors.toList()); } public Iterable<Expression> expressions() { @@ -81,20 +102,19 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) { ArrayList<Expression> later = new ArrayList<>(); - Set<String> touchedFields = new HashSet<String>(); - for (Expression exp : expressions) { + Set<String> touchedFields = new HashSet<>(); + for (Expression expression : expressions) { + if (modifiesSelf(expression) && ! setsLanguage(expression)) + later.add(expression); + else + ilscriptBuilder.content(expression.toString()); + FieldScanVisitor fieldFetcher = new FieldScanVisitor(); - if (modifiesSelf(exp)) { - later.add(exp); - } else { - ilscriptBuilder.content(exp.toString()); - } - fieldFetcher.visit(exp); + fieldFetcher.visit(expression); touchedFields.addAll(fieldFetcher.touchedFields()); } - for (Expression exp : later) { + for (Expression exp : later) ilscriptBuilder.content(exp.toString()); - } generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields); } @@ -110,13 +130,20 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro } } - private boolean modifiesSelf(Expression exp) { - MyExpVisitor visitor = new MyExpVisitor(); - visitor.visit(exp); + private boolean setsLanguage(Expression expression) { + SetsLanguageVisitor visitor = new SetsLanguageVisitor(); + visitor.visit(expression); + return visitor.setsLanguage; + } + + private boolean modifiesSelf(Expression expression) { + ModifiesSelfVisitor visitor = new ModifiesSelfVisitor(); + visitor.visit(expression); return visitor.modifiesSelf(); } - private class MyExpVisitor extends ExpressionVisitor { + private static class ModifiesSelfVisitor extends ExpressionVisitor { + private String inputField = null; private String outputField = null; @@ -124,9 +151,8 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro @Override protected void doVisit(Expression expression) { - if (modifiesSelf()) { - return; - } + if (modifiesSelf()) return; + if (expression instanceof InputExpression) { inputField = ((InputExpression) expression).getFieldName(); } @@ -136,6 +162,18 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro } } + private static class SetsLanguageVisitor extends ExpressionVisitor { + + boolean setsLanguage = false; + + @Override + protected void doVisit(Expression expression) { + if (expression instanceof SetLanguageExpression) + setsLanguage = true; + } + + } + private static class FieldScanVisitor extends ExpressionVisitor { List<String> touchedFields = new ArrayList<String>(); List<String> candidates = new ArrayList<String>(); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index e36635ba6b8..c3a0abc892c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -465,9 +465,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } @Override - public ScriptExpression getIndexingScript() { - return indexingScript; - } + public ScriptExpression getIndexingScript() { return indexingScript; } @SuppressWarnings("deprecation") @Override @@ -477,7 +475,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, stemming=Stemming.NONE; } this.dataType = type; - if (!idOverride) { + if ( ! idOverride) { this.fieldId = calculateIdV7(null); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java index 8f2a29abcb6..416d956ac21 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java @@ -29,7 +29,7 @@ public class OptimizeIlscript extends Processor { field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script)); if ( ! field.getIndexingScript().toString().equals(script.toString())) { - warn(search, field, "Rewrote ilscript from:\n" + script.toString() + + info(search, field, "Rewrote ilscript from:\n" + script.toString() + "\nto\n" + field.getIndexingScript().toString()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 3f225b00277..e8594c2a87f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -27,7 +27,6 @@ public class Processing { IndexFieldNames::new, IntegerIndex2Attribute::new, MakeAliases::new, - SetLanguage::new, UriHack::new, LiteralBoost::new, TagType::new, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java index e15e17817a2..3744af7cc2c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -128,12 +128,22 @@ public abstract class Processor { throw newProcessException(search, field, msg); } - protected void warn(String searchName, String fieldName, String msg) { - String fullMsg = formatError(searchName, fieldName, msg); + protected void warn(String searchName, String fieldName, String message) { + String fullMsg = formatError(searchName, fieldName, message); deployLogger.log(Level.WARNING, fullMsg); } - protected void warn(Search search, Field field, String msg) { - warn(search.getName(), field.getName(), msg); + protected void warn(Search search, Field field, String message) { + warn(search.getName(), field.getName(), message); } + + protected void info(String searchName, String fieldName, String message) { + String fullMsg = formatError(searchName, fieldName, message); + deployLogger.log(Level.INFO, fullMsg); + } + + protected void info(Search search, Field field, String message) { + warn(search.getName(), field.getName(), message); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java deleted file mode 100644 index 8a4795c4dd2..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Search; -import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; -import com.yahoo.vespa.model.container.search.QueryProfiles; - -import java.util.ArrayList; -import java.util.List; - -/** - * Check that no text field appears before a field that sets language. - * - * @author Gunnar Gauslaa Bergem - */ -public class SetLanguage extends Processor { - - public SetLanguage(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { - super(search, deployLogger, rankProfileRegistry, queryProfiles); - } - - @Override - public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - List<String> textFieldsWithoutLanguage = new ArrayList<>(); - - for (SDField field : search.allConcreteFields()) { - if (fieldMustComeAfterLanguageSettingField(field)) { - textFieldsWithoutLanguage.add(field.getName()); - } - if (field.containsExpression(SetLanguageExpression.class) && !textFieldsWithoutLanguage.isEmpty()) { - StringBuilder fieldString = new StringBuilder(); - for (String fieldName : textFieldsWithoutLanguage) { - fieldString.append(fieldName).append(" "); - } - warn(search, field, "Field '" + field.getName() + "' sets the language for this document, " + - "and should be defined as the first field in the searchdefinition." + - "Preceding text fields that will not have their language set: " + - fieldString.toString() + - " (This warning is omitted for any subsequent fields that also do set_language.)"); - return; - } - } - } - - private boolean fieldMustComeAfterLanguageSettingField(SDField field) { - return (!field.containsExpression(SetLanguageExpression.class) && - (field.getDataType() == DataType.STRING)); - } - -} |