diff options
10 files changed, 97 insertions, 59 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java index deed576dc31..48bc972f7ef 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -100,7 +100,7 @@ public class DerivedConfiguration implements AttributesConfig.Producer { queryProfiles, importedModels, deployProperties, executor); indexingScript = new IndexingScript(schema); indexInfo = new IndexInfo(schema); - schemaInfo = new SchemaInfo(schema, rankProfileRegistry); + schemaInfo = new SchemaInfo(schema, rankProfileRegistry, summaries, summaryMap); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java index 232d80d59b4..39432fd0049 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java @@ -6,19 +6,19 @@ import com.yahoo.vespa.documentmodel.SummaryTransform; /** * The result transformation of a named field * - * @author bratseth + * @author bratseth */ public class FieldResultTransform { - private String fieldName; + private final String fieldName; private SummaryTransform transform; - private String argument; + private final String argument; - public FieldResultTransform(String fieldName,SummaryTransform transform,String argument) { - this.fieldName=fieldName; - this.transform=transform; + public FieldResultTransform(String fieldName, SummaryTransform transform, String argument) { + this.fieldName = fieldName; + this.transform = transform; this.argument = argument; } @@ -26,18 +26,19 @@ public class FieldResultTransform { public SummaryTransform getTransform() { return transform; } - public void setTransform(SummaryTransform transform) { this.transform=transform; } + public void setTransform(SummaryTransform transform) { this.transform = transform; } /** Returns the argument of this (used as input to the backend docsum rewriter) */ public String getArgument() { return argument; } public int hashCode() { - return fieldName.hashCode() + 11*transform.hashCode() + 17* argument.hashCode(); + return fieldName.hashCode() + 11 * transform.hashCode() + 17 * argument.hashCode(); } + @Override public boolean equals(Object o) { if (! (o instanceof FieldResultTransform)) return false; - FieldResultTransform other=(FieldResultTransform)o; + FieldResultTransform other = (FieldResultTransform)o; return this.fieldName.equals(other.fieldName) && @@ -45,10 +46,12 @@ public class FieldResultTransform { this.argument.equals(other.argument); } + @Override public String toString() { - String sourceString=""; + String sourceString = ""; if ( ! argument.equals(fieldName)) - sourceString=" (argument: " + argument + ")"; + sourceString = " (argument: " + argument + ")"; return "field " + fieldName + ": " + transform + sourceString; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java index d116d5039ae..f3def5300aa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java @@ -7,6 +7,8 @@ import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.search.SearchCluster; import java.util.Collection; @@ -27,16 +29,22 @@ public final class SchemaInfo implements SchemaInfoConfig.Producer { // The rank profile registry itself is not kept around due to its size. private final Map<String, RankProfileInfo> rankProfiles; - public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry) { + private final Summaries summaries; + private final SummaryMap summaryMap; + + public SchemaInfo(Schema schema, RankProfileRegistry rankProfileRegistry, + Summaries summaries, SummaryMap summaryMap) { this.schema = schema; this.rankProfiles = Collections.unmodifiableMap(toRankProfiles(rankProfileRegistry.rankProfilesOf(schema))); + this.summaries = summaries; + this.summaryMap = summaryMap; } - public String name() {return schema.getName();} + public String name() { return schema.getName(); } - public Schema fullSchema() {return schema;} + public Schema fullSchema() { return schema; } - public Map<String, RankProfileInfo> rankProfiles() {return rankProfiles;} + public Map<String, RankProfileInfo> rankProfiles() { return rankProfiles; } private Map<String, RankProfileInfo> toRankProfiles(Collection<RankProfile> rankProfiles) { Map<String, RankProfileInfo> rankProfileInfos = new LinkedHashMap<>(); @@ -70,12 +78,30 @@ public final class SchemaInfo implements SchemaInfoConfig.Producer { docDb.summaryclass(sumClassBuilder); } */ - /* - for (var summary : schema.getSummaries().values()) { + for (var summary : summaries.asList()) { var summaryBuilder = new SchemaInfoConfig.Schema.Summaryclass.Builder(); summaryBuilder.id(summary.id()).name(summary.getName()); + for (var field : summary.fields().values()) { + var fieldsBuilder = new SchemaInfoConfig.Schema.Summaryclass.Fields.Builder(); + fieldsBuilder.name(field.getName()) + .type(field.getType().getName()) + .dynamic(isDynamic(field.getName())); + summaryBuilder.fields(fieldsBuilder); + } + schemaBuilder.summaryclass(summaryBuilder); } - */ + } + + /** Returns whether the given field is a dynamic summary field. */ + private boolean isDynamic(String fieldName) { + if (summaryMap == null) return false; // not know for streaming, but also not used + + var fieldTransform = summaryMap.resultTransforms().get(fieldName); + if (fieldTransform == null) return false; + // TODO: Move this into SummaryTransform and call it something else than "dynamic" + return fieldTransform.getTransform().isDynamic() || + fieldTransform.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || + fieldTransform.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER; } private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java index 64e1dafd787..e259c07dbb5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java @@ -6,6 +6,8 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.searchdefinition.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.config.search.SummaryConfig; + +import java.util.ArrayList; import java.util.List; /** @@ -16,19 +18,24 @@ import java.util.List; public class Summaries extends Derived implements SummaryConfig.Producer { private final boolean useV8GeoPositions; - private final List<SummaryClass> summaries = new java.util.ArrayList<>(1); + private final List<SummaryClass> summaries; public Summaries(Schema schema, DeployLogger deployLogger, ModelContext.FeatureFlags featureFlags) { super(); this.useV8GeoPositions = featureFlags.useV8GeoPositions(); + // Make sure the default is first + List<SummaryClass> summaries = new ArrayList<>(); summaries.add(new SummaryClass(schema, schema.getSummary("default"), deployLogger)); for (DocumentSummary summary : schema.getSummaries().values()) { if (!summary.getName().equals("default")) summaries.add(new SummaryClass(schema, summary, deployLogger)); } + this.summaries = List.copyOf(summaries); } + public List<SummaryClass> asList() { return summaries; } + @Override protected String getDerivedName() { return "summary"; } @@ -40,4 +47,5 @@ public class Summaries extends Derived implements SummaryConfig.Producer { builder.classes(summaryClass.getSummaryClassConfig()); } } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java index 918f2bd3eda..ea8e2aac1f0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -54,6 +54,8 @@ public class SummaryClass extends Derived { this.id = deriveId(summary.getName(), fields); } + public int id() { return id; } + /** MUST be called after all other fields are added */ private void deriveImplicitFields(DocumentSummary summary, Map<String, SummaryClassField> fields) { if (summary.getName().equals("default")) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java index c65ed7dc762..9e1740b4073 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java @@ -14,13 +14,13 @@ import java.util.Map; /** * A summary map (describing search-time summary field transformations) - * derived from a search definition + * derived from a Schema. * - * @author bratseth + * @author bratseth */ public class SummaryMap extends Derived implements SummarymapConfig.Producer { - private Map<String,FieldResultTransform> resultTransforms = new java.util.LinkedHashMap<>(); + private final Map<String, FieldResultTransform> resultTransforms = new java.util.LinkedHashMap<>(); /** Creates a summary map from a search definition */ SummaryMap(Schema schema) { @@ -49,32 +49,32 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { summaryField.getTransform()==SummaryTransform.MATCHED_ELEMENTS_FILTER || summaryField.getTransform()==SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) { - resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - summaryField.getSingleSource())); + resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getSingleSource())); } else { // Note: Currently source mapping is handled in the indexing statement, // by creating a summary field for each of the values // This works, but is suboptimal. We could consolidate to a minimal set and // use the right value from the minimal set as the third parameter here, // and add "override" commands to multiple static values - resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(), - summaryField.getTransform(), - summaryField.getName())); + resultTransforms.put(summaryField.getName(), new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getName())); } } } /** Returns a read-only iterator of the FieldResultTransforms of this summary map */ - public Iterator resultTransformIterator() { - return Collections.unmodifiableCollection(resultTransforms.values()).iterator(); + public Map<String, FieldResultTransform> resultTransforms() { + return Collections.unmodifiableMap(resultTransforms); } protected String getDerivedName() { return "summarymap"; } /** Returns the command name of a transform */ private String getCommand(SummaryTransform transform) { - if (transform.equals(SummaryTransform.DISTANCE)) + if (transform == SummaryTransform.DISTANCE) return "absdist"; else if (transform.isDynamic()) return "dynamicteaser"; @@ -87,6 +87,7 @@ public class SummaryMap extends Derived implements SummarymapConfig.Producer { * We need this because some model information is shared through configs instead of model - see usage * A dynamic transform needs the query to perform its computations. */ + // TODO/Note: "dynamic" here means something else than in SummaryTransform public static boolean isDynamicCommand(String commandName) { return (commandName.equals("dynamicteaser") || commandName.equals(SummaryTransform.MATCHED_ELEMENTS_FILTER.getName()) || diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index d814f3e5d63..c50766a2585 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -24,7 +24,7 @@ public enum SummaryTransform { MATCHED_ELEMENTS_FILTER("matchedelementsfilter"), MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"); - private String name; + private final String name; SummaryTransform(String name) { this.name=name; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index fab5bd627f3..82321216519 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -243,8 +243,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> throw new IllegalArgumentException("Schema '" + schemaDefinitionXMLHandler.getName() + "' referenced in " + this + " does not exist"); - sc.add(new SchemaInfo(schema, - deployState.rankProfileRegistry())); + sc.add(new SchemaInfo(schema, deployState.rankProfileRegistry(), null, null)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java index 7394670cb1e..e628907068d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java @@ -87,7 +87,6 @@ public class DocumentDatabase extends AbstractConfigProducer<DocumentDatabase> i @Override public void getConfig(SummarymapConfig.Builder builder) { derivedCfg.getSummaryMap().getConfig(builder); } - @Override public void getConfig(SummaryConfig.Builder builder) { derivedCfg.getSummaries().getConfig(builder); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java index bfa135de90e..6abe0e763cb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java @@ -31,44 +31,44 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { @Test public void testDeriving() throws IOException, ParseException { Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/simple.sd"); - SummaryMap summaryMap=new SummaryMap(schema); + SummaryMap summaryMap = new SummaryMap(schema); - Iterator transforms=summaryMap.resultTransformIterator(); - FieldResultTransform transform = (FieldResultTransform)transforms.next(); + Iterator<FieldResultTransform> transforms = summaryMap.resultTransforms().values().iterator(); + FieldResultTransform transform = transforms.next(); assertEquals("dyndesc", transform.getFieldName()); - assertEquals(SummaryTransform.DYNAMICTEASER,transform.getTransform()); + assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("dynlong", transform.getFieldName()); - assertEquals(SummaryTransform.DYNAMICTEASER,transform.getTransform()); + assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("dyndesc2", transform.getFieldName()); - assertEquals(SummaryTransform.DYNAMICTEASER,transform.getTransform()); + assertEquals(SummaryTransform.DYNAMICTEASER, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("measurement", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("rankfeatures", transform.getFieldName()); assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("summaryfeatures", transform.getFieldName()); assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("popsiness", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("popularity", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("access", transform.getFieldName()); - assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); + assertEquals(SummaryTransform.ATTRIBUTE, transform.getTransform()); assertFalse(transforms.hasNext()); } @@ -84,22 +84,22 @@ public class SummaryMapTestCase extends AbstractSchemaTestCase { true, false, Set.of()); SummaryMap summaryMap = new SummaryMap(schema); - Iterator transforms = summaryMap.resultTransformIterator(); + Iterator<FieldResultTransform> transforms = summaryMap.resultTransforms().values().iterator(); - FieldResultTransform transform = (FieldResultTransform)transforms.next(); + FieldResultTransform transform = transforms.next(); assertEquals(fieldName, transform.getFieldName()); assertEquals(SummaryTransform.GEOPOS, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("rankfeatures", transform.getFieldName()); assertEquals(SummaryTransform.RANKFEATURES, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("summaryfeatures", transform.getFieldName()); assertEquals(SummaryTransform.SUMMARYFEATURES, transform.getTransform()); - transform = (FieldResultTransform)transforms.next(); + transform = transforms.next(); assertEquals("location_zcurve", transform.getFieldName()); assertEquals(SummaryTransform.ATTRIBUTE,transform.getTransform()); |