diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo/searchdefinition')
8 files changed, 216 insertions, 81 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java index 3a8268029d0..99c40c2700f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java @@ -26,9 +26,17 @@ public abstract class Derived implements Exportable { private String name; + public Derived() { + this(""); + } + + public Derived(String name) { + this.name = name; + } + public String getName() { return name; } - protected final void setName(String name) { this.name=name; } + protected final void setName(String name) { this.name = name; } /** * Derives the content of this configuration. This diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index 64a679ed454..48bc972f7ef 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -24,7 +24,7 @@ import java.io.Writer; import java.util.concurrent.ExecutorService; /** - * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when + * A set of all derived configuration of a schema. Use this as a facade to individual configurations when * necessary. * * @author bratseth @@ -39,6 +39,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { private RankProfileList rankProfileList; private IndexingScript indexingScript; private IndexInfo indexInfo; + private SchemaInfo schemaInfo; private VsmFields streamingFields; private VsmSummary streamingSummary; private IndexSchema indexSchema; @@ -99,6 +100,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { queryProfiles, importedModels, deployProperties, executor); indexingScript = new IndexingScript(schema); indexInfo = new IndexInfo(schema); + schemaInfo = new SchemaInfo(schema, rankProfileRegistry, summaries, summaryMap); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); } @@ -179,6 +181,8 @@ public class DerivedConfiguration implements AttributesConfig.Producer { return indexInfo; } + public SchemaInfo getSchemaInfo() { return schemaInfo; } + public void setIndexingScript(IndexingScript script) { this.indexingScript = script; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java index 232d80d59b4..39432fd0049 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java @@ -6,19 +6,19 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; /** * The result transformation of a named field * - * @author bratseth + * @author bratseth */ public class FieldResultTransform { - private String fieldName; + private final String fieldName; private SummaryTransform transform; - private String argument; + private final String argument; - public FieldResultTransform(String fieldName,SummaryTransform transform,String argument) { - this.fieldName=fieldName; - this.transform=transform; + public FieldResultTransform(String fieldName, SummaryTransform transform, String argument) { + this.fieldName = fieldName; + this.transform = transform; this.argument = argument; } @@ -26,18 +26,19 @@ public class FieldResultTransform { public SummaryTransform getTransform() { return transform; } - public void setTransform(SummaryTransform transform) { this.transform=transform; } + public void setTransform(SummaryTransform transform) { this.transform = transform; } /** Returns the argument of this (used as input to the backend docsum rewriter) */ public String getArgument() { return argument; } public int hashCode() { - return fieldName.hashCode() + 11*transform.hashCode() + 17* argument.hashCode(); + return fieldName.hashCode() + 11 * transform.hashCode() + 17 * argument.hashCode(); } + @Override public boolean equals(Object o) { if (! (o instanceof FieldResultTransform)) return false; - FieldResultTransform other=(FieldResultTransform)o; + FieldResultTransform other = (FieldResultTransform)o; return this.fieldName.equals(other.fieldName) && @@ -45,10 +46,12 @@ public class FieldResultTransform { this.argument.equals(other.argument); } + @Override public String toString() { - String sourceString=""; + String sourceString = ""; if ( ! argument.equals(fieldName)) - sourceString=" (argument: " + argument + ")"; + sourceString = " (argument: " + argument + ")"; return "field " + fieldName + ": " + transform + sourceString; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index bcebdf3a916..cd94b4a7f6e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -41,7 +41,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ private final LargeRankExpressions largeRankExpressions; private final OnnxModels onnxModels; - public static RankProfileList empty = new RankProfileList(); + public static final RankProfileList empty = new RankProfileList(); private RankProfileList() { rankingConstants = new RankingConstants(null, Optional.empty()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java new file mode 100644 index 00000000000..3698345eb11 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java @@ -0,0 +1,129 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.search.config.SchemaInfoConfig; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.search.SearchCluster; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Information about a schema. + * + * @author bratseth + */ +public final class SchemaInfo implements SchemaInfoConfig.Producer { + + private final Schema schema; + + // Info about profiles needed in memory after build. + // The rank profile registry itself is not kept around due to its size. + private final Map<String, RankProfileInfo> rankProfiles; + + private final Summaries summaries; + private final SummaryMap summaryMap; + + public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry, + Summaries summaries, SummaryMap summaryMap) { + this.schema = schema; + this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema))); + this.summaries = summaries; + this.summaryMap = summaryMap; + } + + public String name() { return schema.getName(); } + + public Schema fullSchema() { return schema; } + + public Map<String, RankProfileInfo> rankProfiles() { return rankProfiles; } + + private Map<String, RankProfileInfo> toRankProfiles(Collection<RankProfile> rankProfiles) { + Map<String, RankProfileInfo> rankProfileInfos = new LinkedHashMap<>(); + rankProfiles.forEach(profile -> rankProfileInfos.put(profile.name(), new RankProfileInfo(profile))); + return rankProfileInfos; + } + + @Override + public void getConfig(SchemaInfoConfig.Builder builder) { + var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); + schemaBuilder.name(schema.getName()); + addSummaryConfig(schemaBuilder); + addRankProfilesConfig(schemaBuilder); + builder.schema(schemaBuilder); + } + + private void addSummaryConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (var summary : summaries.asList()) { + var summaryBuilder = new SchemaInfoConfig.Schema.Summaryclass.Builder(); + summaryBuilder.name(summary.getName()); + for (var field : summary.fields().values()) { + var fieldsBuilder = new SchemaInfoConfig.Schema.Summaryclass.Fields.Builder(); + fieldsBuilder.name(field.getName()) + .type(field.getType().getName()) + .dynamic(isDynamic(field.getName())); + summaryBuilder.fields(fieldsBuilder); + } + schemaBuilder.summaryclass(summaryBuilder); + } + } + + /** Returns whether the given field is a dynamic summary field. */ + private boolean isDynamic(String fieldName) { + if (summaryMap == null) return false; // not know for streaming, but also not used + + var fieldTransform = summaryMap.resultTransforms().get(fieldName); + if (fieldTransform == null) return false; + // TODO: Move this into SummaryTransform and call it something else than "dynamic" + return fieldTransform.getTransform().isDynamic() || + fieldTransform.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || + fieldTransform.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER; + } + + private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { + for (RankProfileInfo rankProfile : rankProfiles().values()) { + var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); + rankProfileConfig.name(rankProfile.name()); + rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); + rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); + for (var input : rankProfile.inputs().entrySet()) { + var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); + inputConfig.name(input.getKey().toString()); + inputConfig.type(input.getValue().toString()); + rankProfileConfig.input(inputConfig); + } + schemaBuilder.rankprofile(rankProfileConfig); + } + } + + /** A store of a *small* (in memory) amount of rank profile info. */ + public static final class RankProfileInfo { + + private final String name; + private final boolean hasSummaryFeatures; + private final boolean hasRankFeatures; + private final Map<Reference, TensorType> inputs; + + public RankProfileInfo(RankProfile profile) { + this.name = profile.name(); + this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty(); + this.hasRankFeatures = ! profile.getRankFeatures().isEmpty(); + this.inputs = profile.inputs(); + } + + public String name() { return name; } + public boolean hasSummaryFeatures() { return hasSummaryFeatures; } + public boolean hasRankFeatures() { return hasRankFeatures; } + public Map<Reference, TensorType> inputs() { return inputs; } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java index 1455fbc92e1..e259c07dbb5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java @@ -6,6 +6,8 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.config.search.SummaryConfig; + +import java.util.ArrayList; import java.util.List; /** @@ -16,18 +18,24 @@ import java.util.List; public class Summaries extends Derived implements SummaryConfig.Producer { private final boolean useV8GeoPositions; - private List<SummaryClass> summaries=new java.util.ArrayList<>(1); + private final List<SummaryClass> summaries; public Summaries(Schema schema, DeployLogger deployLogger, ModelContext.FeatureFlags featureFlags) { + super(); this.useV8GeoPositions = featureFlags.useV8GeoPositions(); + // Make sure the default is first + List<SummaryClass> summaries = new ArrayList<>(); summaries.add(new SummaryClass(schema, schema.getSummary("default"), deployLogger)); for (DocumentSummary summary : schema.getSummaries().values()) { if (!summary.getName().equals("default")) summaries.add(new SummaryClass(schema, summary, deployLogger)); } + this.summaries = List.copyOf(summaries); } + public List<SummaryClass> asList() { return summaries; } + @Override protected String getDerivedName() { return "summary"; } @@ -39,4 +47,5 @@ public class Summaries extends Derived implements SummaryConfig.Producer { builder.classes(summaryClass.getSummaryClassConfig()); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java index 6c233aacf30..ea8e2aac1f0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -9,9 +9,9 @@ import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; -import java.util.Iterator; + +import java.util.Collections; import java.util.Map; -import java.util.Random; import java.util.logging.Level; /** @@ -19,63 +19,66 @@ import java.util.logging.Level; * Each summary definition have at least one summary, the default * which has the same name as the search definition. * - * @author bratseth + * @author bratseth */ public class SummaryClass extends Derived { public static final String DOCUMENT_ID_FIELD = "documentid"; + private final int id; + /** True if this summary class needs to access summary information on disk */ private boolean accessingDiskSummary = false; private final boolean rawAsBase64; private final boolean omitSummaryFeatures; /** The summary fields of this indexed by name */ - private final Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); + private final Map<String, SummaryClassField> fields; private final DeployLogger deployLogger; - private final Random random = new Random(7); - /** * Creates a summary class from a search definition summary * * @param deployLogger a {@link DeployLogger} */ public SummaryClass(Schema schema, DocumentSummary summary, DeployLogger deployLogger) { + super(summary.getName()); this.deployLogger = deployLogger; this.rawAsBase64 = schema.isRawAsBase64(); this.omitSummaryFeatures = summary.omitSummaryFeatures(); - deriveName(summary); - deriveFields(schema, summary); - deriveImplicitFields(summary); + Map<String, SummaryClassField> fields = new java.util.LinkedHashMap<>(); + deriveFields(schema, summary, fields); + deriveImplicitFields(summary, fields); + this.fields = Collections.unmodifiableMap(fields); + this.id = deriveId(summary.getName(), fields); } - private void deriveName(DocumentSummary summary) { - setName(summary.getName()); - } + public int id() { return id; } /** MUST be called after all other fields are added */ - private void deriveImplicitFields(DocumentSummary summary) { + private void deriveImplicitFields(DocumentSummary summary, Map<String, SummaryClassField> fields) { if (summary.getName().equals("default")) { - addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING); + addField(SummaryClass.DOCUMENT_ID_FIELD, DataType.STRING, fields); } } - private void deriveFields(Schema schema, DocumentSummary summary) { + private void deriveFields(Schema schema, DocumentSummary summary, Map<String, SummaryClassField> fields) { for (SummaryField summaryField : summary.getSummaryFields().values()) { if (!accessingDiskSummary && schema.isAccessingDiskSummary(summaryField)) { accessingDiskSummary = true; } - addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform()); + addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform(), fields); } } - private void addField(String name, DataType type) { - addField(name, type, null); + private void addField(String name, DataType type, Map<String, SummaryClassField> fields) { + addField(name, type, null, fields); } - private void addField(String name, DataType type, SummaryTransform transform) { + private void addField(String name, DataType type, + SummaryTransform transform, + Map<String, SummaryClassField> fields) { if (fields.containsKey(name)) { SummaryClassField sf = fields.get(name); if ( SummaryClassField.convertDataType(type, transform, rawAsBase64) != sf.getType()) { @@ -87,48 +90,23 @@ public class SummaryClass extends Derived { } } + public Map<String, SummaryClassField> fields() { return fields; } - /** Returns an iterator of the fields of this summary. Removes on this iterator removes the field from this summary */ - public Iterator<SummaryClassField> fieldIterator() { - return fields.values().iterator(); - } - - public void addField(SummaryClassField field) { - fields.put(field.getName(),field); - } - - /** Returns the writable map of fields of this summary */ // TODO: Make read only, move writers to iterator/addField - public Map<String,SummaryClassField> getFields() { return fields; } - - public SummaryClassField getField(String name) { - return fields.get(name); - } - - public int getFieldCount() { return fields.size(); } - - @Override - public int hashCode() { + private static int deriveId(String name, Map<String, SummaryClassField> fields) { + int hash = name.hashCode(); int number = 1; - int hash = getName().hashCode(); - for (Iterator i = fieldIterator(); i.hasNext(); ) { - SummaryClassField field = (SummaryClassField)i.next(); - hash += number * (field.getName().hashCode() + - 17*field.getType().getName().hashCode()); - number++; + for (var field : fields.values()) { + hash += number++ * (field.getName().hashCode() + + 17 * field.getType().getName().hashCode()); } - if (hash < 0) - hash *= -1; + hash = Math.abs(hash); + if (hash == DocsumDefinitionSet.SLIME_MAGIC_ID) + hash++; return hash; } public SummaryConfig.Classes.Builder getSummaryClassConfig() { SummaryConfig.Classes.Builder classBuilder = new SummaryConfig.Classes.Builder(); - int id = hashCode(); - if (id == DocsumDefinitionSet.SLIME_MAGIC_ID) { - deployLogger.log(Level.WARNING, "Summary class '" + getName() + "' hashes to the SLIME_MAGIC_ID '" + id + - "'. This is unlikely but I autofix it for you by adding a random number."); - id += random.nextInt(); - } classBuilder. id(id). name(getName()). @@ -142,11 +120,14 @@ public class SummaryClass extends Derived { } @Override + public int hashCode() { return id; } + + @Override protected String getDerivedName() { return "summary"; } @Override public String toString() { - return "summary class " + getName(); + return "summary class '" + getName() + "'"; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java index c65ed7dc762..9e1740b4073 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java @@ -14,13 +14,13 @@ import java.util.Map; /** * A summary map (describing search-time summary field transformations) - * derived from a search definition + * derived from a Schema. * - * @author bratseth + * @author bratseth */ public class SummaryMap extends Derived implements SummarymapConfig.Producer { - private Map<String,FieldResultTransform> resultTransforms = new java.util.LinkedHashMap<>(); + private final Map<String, FieldResultTransform> resultTransforms = new java.util.LinkedHashMap<>(); /** Creates a summary map from a search definition */ SummaryMap(Schema schema) { @@ -49,32 +49,32 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { summaryField.getTransform()==SummaryTransform.MATCHED_ELEMENTS_FILTER || summaryField.getTransform()==SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) { - resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - summaryField.getSingleSource())); + resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getSingleSource())); } else { // Note: Currently source mapping is handled in the indexing statement, // by creating a summary field for each of the values // This works, but is suboptimal. We could consolidate to a minimal set and // use the right value from the minimal set as the third parameter here, // and add "override" commands to multiple static values - resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - summaryField.getName())); + resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getName())); } } } /** Returns a read-only iterator of the FieldResultTransforms of this summary map */ - public Iterator resultTransformIterator() { - return Collections.unmodifiableCollection(resultTransforms.values()).iterator(); + public Map<String, FieldResultTransform> resultTransforms() { + return Collections.unmodifiableMap(resultTransforms); } protected String getDerivedName() { return "summarymap"; } /** Returns the command name of a transform */ private String getCommand(SummaryTransform transform) { - if (transform.equals(SummaryTransform.DISTANCE)) + if (transform == SummaryTransform.DISTANCE) return "absdist"; else if (transform.isDynamic()) return "dynamicteaser"; @@ -87,6 +87,7 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { * We need this because some model information is shared through configs instead of model - see usage * A dynamic transform needs the query to perform its computations. */ + // TODO/Note: "dynamic" here means something else than in SummaryTransform public static boolean isDynamicCommand(String commandName) { return (commandName.equals("dynamicteaser") || commandName.equals(SummaryTransform.MATCHED_ELEMENTS_FILTER.getName()) || |