Rename the 'searchdefinition' package to 'schema'

5 files changed, 289 insertions, 0 deletions

diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java
new file mode 100644
index 00000000000..565a377f2a9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java
@@ -0,0 +1,62 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.schema.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.schema.document.SDField;
+import com.yahoo.schema.Schema;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Resolver-class for harmonizing index-commands in multifield indexes
+ */
+public class IndexCommandResolver extends MultiFieldResolver {
+
+    /** Commands which don't have to be harmonized between fields */
+    private static List<String> ignoredCommands = new ArrayList<>();
+
+    /** Commands which must be harmonized between fields */
+    private static List<String> harmonizedCommands = new ArrayList<>();
+
+    static {
+        String[] ignore = { "complete-boost", "literal-boost", "highlight" };
+        ignoredCommands.addAll(Arrays.asList(ignore));
+        String[] harmonize = { "stemming", "normalizing" };
+        harmonizedCommands.addAll(Arrays.asList(harmonize));
+    }
+
+    public IndexCommandResolver(String indexName, List<SDField> fields, Schema schema, DeployLogger logger) {
+        super(indexName, fields, schema, logger);
+    }
+
+    /**
+     * Check index-commands for each field, report and attempt to fix any
+     * inconsistencies
+     */
+    public void resolve() {
+        for (SDField field : fields) {
+            for (String command : field.getQueryCommands()) {
+                if (!ignoredCommands.contains(command))
+                    checkCommand(command);
+            }
+        }
+    }
+
+    private void checkCommand(String command) {
+        for (SDField field : fields) {
+            if (!field.hasQueryCommand(command)) {
+                if (harmonizedCommands.contains(command)) {
+                    deployLogger.logApplicationPackage(Level.WARNING, command + " must be added to all fields going to the same index (" + indexName + ")" +
+                            ", adding to field " + field.getName());
+                    field.addQueryCommand(command);
+                } else {
+                    deployLogger.logApplicationPackage(Level.WARNING, "All fields going to the same index should have the same query-commands. Field \'" + field.getName() +
+                            "\' doesn't contain command \'" + command+"\'");
+                }
+            }
+        }
+    }
+}
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/MultiFieldResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/MultiFieldResolver.java
new file mode 100644
index 00000000000..ed8ad61706b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/MultiFieldResolver.java
@@ -0,0 +1,33 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.schema.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.schema.document.SDField;
+import com.yahoo.schema.Schema;
+import java.util.List;
+
+/**
+ * Abstract superclass of all multifield conflict resolvers
+ */
+public abstract class MultiFieldResolver {
+
+    protected String indexName;
+    protected List<SDField> fields;
+    protected Schema schema;
+
+    protected DeployLogger deployLogger;
+
+    public MultiFieldResolver(String indexName, List<SDField> fields, Schema schema, DeployLogger logger) {
+        this.indexName = indexName;
+        this.fields = fields;
+        this.schema = schema;
+        this.deployLogger = logger;
+    }
+
+    /**
+     * Checks the list of fields for specific conflicts, and reports and/or
+     * attempts to correct them
+     */
+    public abstract void resolve();
+
+}
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
new file mode 100644
index 00000000000..3d79ac7d68a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java
@@ -0,0 +1,105 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.schema.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.search.query.profile.types.FieldDescription;
+import com.yahoo.search.query.profile.types.FieldType;
+import com.yahoo.search.query.profile.types.QueryProfileType;
+import com.yahoo.search.query.profile.types.TensorFieldType;
+import com.yahoo.schema.FeatureNames;
+import com.yahoo.schema.RankProfile;
+import com.yahoo.schema.RankProfileRegistry;
+import com.yahoo.schema.Schema;
+import com.yahoo.schema.document.Attribute;
+import com.yahoo.schema.document.ImmutableSDField;
+import com.yahoo.schema.document.ImportedField;
+import com.yahoo.schema.document.ImportedFields;
+import com.yahoo.schema.processing.Processor;
+import com.yahoo.searchlib.rankingexpression.Reference;
+import com.yahoo.tensor.TensorType;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * This processes a schema and adds input type settings on all rank profiles.
+ *
+ * Currently, type settings are limited to the type of tensor attribute fields and tensor query features.
+ *
+ * @author geirst
+ */
+public class RankProfileTypeSettingsProcessor extends Processor {
+
+    public RankProfileTypeSettingsProcessor(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) {
+        super(schema, deployLogger, rankProfileRegistry, queryProfiles);
+    }
+
+    @Override
+    public void process(boolean validate, boolean documentsOnly) {
+        if (documentsOnly) return;
+
+        processAttributeFields();
+        processImportedFields();
+        processQueryProfileTypes();
+    }
+
+    private void processAttributeFields() {
+        if (schema == null) return; // we're processing global profiles
+        for (ImmutableSDField field : schema.allConcreteFields()) {
+            Attribute attribute = field.getAttributes().get(field.getName());
+            if (attribute != null && attribute.tensorType().isPresent()) {
+                addAttributeTypeToRankProfiles(attribute.getName(), attribute.tensorType().get().toString());
+            }
+        }
+    }
+
+    private void processImportedFields() {
+        if (schema == null) return; // we're processing global profiles
+        Optional<ImportedFields> importedFields = schema.importedFields();
+        if (importedFields.isPresent()) {
+            importedFields.get().fields().forEach((fieldName, field) -> processImportedField(field));
+        }
+    }
+
+    private void processImportedField(ImportedField field) {
+        ImmutableSDField targetField = field.targetField();
+        Attribute attribute = targetField.getAttributes().get(targetField.getName());
+        if (attribute != null && attribute.tensorType().isPresent()) {
+            addAttributeTypeToRankProfiles(field.fieldName(), attribute.tensorType().get().toString());
+        }
+    }
+
+    private void addAttributeTypeToRankProfiles(String attributeName, String attributeType) {
+        for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) {
+            profile.addAttributeType(attributeName, attributeType);
+        }
+    }
+
+    private void processQueryProfileTypes() {
+        for (QueryProfileType queryProfileType : queryProfiles.getRegistry().getTypeRegistry().allComponents()) {
+            for (Map.Entry<String, FieldDescription> fieldDescEntry : queryProfileType.fields().entrySet()) {
+                processFieldDescription(fieldDescEntry.getValue());
+            }
+        }
+    }
+
+    private void processFieldDescription(FieldDescription fieldDescription) {
+        FieldType fieldType = fieldDescription.getType();
+        if (fieldType instanceof TensorFieldType) {
+            TensorFieldType tensorFieldType = (TensorFieldType)fieldType;
+            Optional<Reference> reference = Reference.simple(fieldDescription.getName());
+            if (reference.isPresent() && FeatureNames.isQueryFeature(reference.get()))
+                addQueryFeatureTypeToRankProfiles(reference.get(), tensorFieldType.asTensorType());
+        }
+    }
+
+    private void addQueryFeatureTypeToRankProfiles(Reference queryFeature, TensorType queryFeatureType) {
+        for (RankProfile profile : rankProfileRegistry.all()) {
+            if (! profile.inputs().containsKey(queryFeature)) // declared inputs have precedence
+                profile.addInput(queryFeature,
+                                 new RankProfile.Input(queryFeature, queryFeatureType, Optional.empty()));
+        }
+    }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java
new file mode 100644
index 00000000000..6424fd8ba06
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java
@@ -0,0 +1,46 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.schema.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.schema.document.RankType;
+import com.yahoo.schema.document.SDField;
+import com.yahoo.schema.Schema;
+
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Checks if fields have defined different rank types for the same
+ * index (typically in an index-to statement), and if they have
+ * output a warning and use the first ranktype.
+ *
+ * @author hmusum
+ */
+public class RankTypeResolver extends MultiFieldResolver {
+
+    public RankTypeResolver(String indexName, List<SDField> fields, Schema schema, DeployLogger logger) {
+        super(indexName, fields, schema, logger);
+    }
+
+    public void resolve() {
+        RankType rankType = null;
+        if (fields.size() > 0) {
+            boolean first = true;
+            for (SDField field : fields) {
+                if (first) {
+                    rankType = fields.get(0).getRankType();
+                    first = false;
+                } else if (!field.getRankType().equals(rankType)) {
+                    deployLogger.logApplicationPackage(Level.WARNING, "In field '" + field.getName() + "' " +
+                            field.getRankType() + " for index '" + indexName +
+                            "' conflicts with " + rankType +
+                            " defined for the same index in field '" +
+                            field.getName() + "'. Using " +
+                            rankType + ".");
+                    field.setRankType(rankType);
+                }
+            }
+        }
+    }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/StemmingResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/StemmingResolver.java
new file mode 100644
index 00000000000..95d9a50a6ab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/StemmingResolver.java
@@ -0,0 +1,43 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.schema.processing.multifieldresolver;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.schema.Schema;
+import com.yahoo.schema.document.SDField;
+import com.yahoo.schema.document.Stemming;
+
+import java.util.List;
+import java.util.logging.Level;
+
+/**
+ * Class resolving conflicts when fields with different stemming-settings are
+ * combined into the same index
+ */
+public class StemmingResolver extends MultiFieldResolver {
+
+    public StemmingResolver(String indexName, List<SDField> fields, Schema schema, DeployLogger logger) {
+        super(indexName, fields, schema, logger);
+    }
+
+    @Override
+    public void resolve() {
+        checkStemmingForIndexFields(indexName, fields);
+    }
+
+    private void checkStemmingForIndexFields(String indexName, List<SDField> fields) {
+        Stemming stemming = null;
+        SDField stemmingField = null;
+        for (SDField field : fields) {
+            if (stemming == null && stemmingField==null) {
+                stemming = field.getStemming(schema);
+                stemmingField = field;
+            } else if (stemming != field.getStemming(schema)) {
+                deployLogger.logApplicationPackage(Level.WARNING, "Field '" + field.getName() + "' has " + field.getStemming(schema) +
+                                                                  ", whereas field '" + stemmingField.getName() + "' has " + stemming +
+                                                                  ". All fields indexing to the index '" + indexName + "' must have the same stemming." +
+                                                                  " This should be corrected as it will make indexing fail in a few cases.");
+            }
+        }
+    }
+
+}