diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2024-03-07 14:40:57 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-03-07 14:40:57 +0100 |
commit | c08a17db6f2cd7541afd17c1b2d8b9bc0fcf8fd3 (patch) | |
tree | 82baf71a67bd9b0d59d45821840bb1b29f6abb64 /config-model/src/main/java/com/yahoo | |
parent | 0bb0b0219a29f30865655e7b07024429b3453180 (diff) | |
parent | 3ebb5e7dd397cacdd82803255907dc4fcdd0ff5f (diff) |
Merge pull request #30499 from vespa-engine/balder/provide-configid-per-document-type
- Set the complete list of the config where it contains the complete …
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
14 files changed, 90 insertions, 66 deletions
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java index 765ad858535..a781adcefb7 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java @@ -9,6 +9,8 @@ import com.yahoo.schema.document.ImportedComplexField; import com.yahoo.schema.document.ImportedField; import com.yahoo.vespa.config.search.ImportedFieldsConfig; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; @@ -40,8 +42,11 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod @Override public void getConfig(ImportedFieldsConfig.Builder builder) { + // Replace if (importedFields.isPresent()) { - importedFields.get().fields().forEach( (name, field) -> considerField(builder, field)); + List<ImportedField> imported = new ArrayList<>(); + importedFields.get().fields().forEach( (name, field) -> considerField(imported, field)); + builder.attribute(imported.stream().map(ImportedFields::createAttributeBuilder).toList()); } } @@ -49,47 +54,46 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod return fieldName.indexOf('.') != -1; } - private static void considerField(ImportedFieldsConfig.Builder builder, ImportedField field) { + private static void considerField(List<ImportedField> importedFields, ImportedField field) { if (field instanceof ImportedComplexField) { - considerComplexField(builder, (ImportedComplexField) field); + considerComplexField(importedFields, (ImportedComplexField) field); } else { - considerSimpleField(builder, field); + considerSimpleField(importedFields, field); } } - private static void considerComplexField(ImportedFieldsConfig.Builder builder, ImportedComplexField field) { + private static void considerComplexField(List<ImportedField> importedFields, ImportedComplexField field) { ImmutableSDField targetField = field.targetField(); if (GeoPos.isAnyPos(targetField)) { // no action needed } else if (isArrayOfSimpleStruct(targetField)) { - considerNestedFields(builder, field); + considerNestedFields(importedFields, field); } else if (isMapOfSimpleStruct(targetField)) { - considerSimpleField(builder, field.getNestedField("key")); - considerNestedFields(builder, field.getNestedField("value")); + considerSimpleField(importedFields, field.getNestedField("key")); + considerNestedFields(importedFields, field.getNestedField("value")); } else if (isMapOfPrimitiveType(targetField)) { - considerSimpleField(builder, field.getNestedField("key")); - considerSimpleField(builder, field.getNestedField("value")); + considerSimpleField(importedFields, field.getNestedField("key")); + considerSimpleField(importedFields, field.getNestedField("value")); } } - private static void considerNestedFields(ImportedFieldsConfig.Builder builder, ImportedField field) { - if (field instanceof ImportedComplexField) { - ImportedComplexField complexField = (ImportedComplexField) field; - complexField.getNestedFields().forEach(nestedField -> considerSimpleField(builder, nestedField)); + private static void considerNestedFields(List<ImportedField> importedFields, ImportedField field) { + if (field instanceof ImportedComplexField complexField) { + complexField.getNestedFields().forEach(nestedField -> considerSimpleField(importedFields, nestedField)); } } - private static void considerSimpleField(ImportedFieldsConfig.Builder builder, ImportedField field) { + private static void considerSimpleField(List<ImportedField> importedFields, ImportedField field) { ImmutableSDField targetField = field.targetField(); String targetFieldName = targetField.getName(); if (!isNestedFieldName(targetFieldName)) { if (targetField.doesAttributing()) { - builder.attribute.add(createAttributeBuilder(field)); + importedFields.add(field); } } else { Attribute attribute = targetField.getAttribute(); if (attribute != null) { - builder.attribute.add(createAttributeBuilder(field)); + importedFields.add(field); } } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index 34f485b7f02..b0cf4a0013c 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -316,6 +316,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { @Override public void getConfig(IndexInfoConfig.Builder builder) { + // Append IndexInfoConfig.Indexinfo.Builder iiB = new IndexInfoConfig.Indexinfo.Builder(); iiB.name(getName()); for (IndexCommand command : commands) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java index d25ada26c6c..1c22e80e579 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java @@ -109,30 +109,33 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { return "indexschema"; } - @Override - public void getConfig(IndexschemaConfig.Builder icB) { - for (IndexField f : fields) { - IndexschemaConfig.Indexfield.Builder ifB = new IndexschemaConfig.Indexfield.Builder() + private static IndexschemaConfig.Indexfield.Builder createIndexFieldConfig(IndexField f) { + var ifB = new IndexschemaConfig.Indexfield.Builder() .name(f.getName()) .datatype(IndexschemaConfig.Indexfield.Datatype.Enum.valueOf(f.getType())) .prefix(f.hasPrefix()) .phrases(false) .positions(true) .interleavedfeatures(f.useInterleavedFeatures()); - if (!f.getCollectionType().equals("SINGLE")) { - ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType())); - } - icB.indexfield(ifB); - } - for (FieldSet fieldSet : fieldSets.values()) { - IndexschemaConfig.Fieldset.Builder fsB = new IndexschemaConfig.Fieldset.Builder() - .name(fieldSet.getName()); - for (String f : fieldSet.getFieldNames()) { - fsB.field(new IndexschemaConfig.Fieldset.Field.Builder() - .name(f)); - } - icB.fieldset(fsB); + if (!f.getCollectionType().equals("SINGLE")) { + ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType())); } + return ifB; + } + + private static IndexschemaConfig.Fieldset.Builder createFieldSetConfig(FieldSet fieldSet) { + var fsB = new IndexschemaConfig.Fieldset.Builder().name(fieldSet.getName()); + for (String f : fieldSet.getFieldNames()) { + fsB.field(new IndexschemaConfig.Fieldset.Field.Builder().name(f)); + } + return fsB; + } + + @Override + public void getConfig(IndexschemaConfig.Builder icB) { + // Replace + icB.indexfield(fields.stream().map(IndexSchema::createIndexFieldConfig).toList()); + icB.fieldset(fieldSets.values().stream().map(IndexSchema::createFieldSetConfig).toList()); } static List<Field> flattenField(Field field) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java index 9f41f4d3542..7245e3f266d 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java @@ -95,6 +95,7 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro @Override public void getConfig(IlscriptsConfig.Builder configBuilder) { + // Append IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder(); ilscriptBuilder.doctype(getName()); ilscriptBuilder.docfield(docFields); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java index ab1bd256ef9..7eefb9e3396 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java @@ -44,18 +44,21 @@ public class Juniperrc extends Derived implements JuniperrcConfig.Producer { @Override protected String getDerivedName() { return "juniperrc"; } + private static JuniperrcConfig.Override.Builder createOverride(String name) { + return new JuniperrcConfig.Override.Builder() + .fieldname(name) + .length(64*Mb) + .max_matches(1) + .min_length(8192) + .surround_max(64*Mb); + } + @Override public void getConfig(JuniperrcConfig.Builder builder) { - if (boldingFields.size() != 0) { + // Replace + if (!boldingFields.isEmpty()) { builder.prefix(true); - for (String name : boldingFields) { - builder.override(new JuniperrcConfig.Override.Builder() - .fieldname(name) - .length(64*Mb) - .max_matches(1) - .min_length(8192) - .surround_max(64*Mb)); - } + builder.override(boldingFields.stream().map(Juniperrc::createOverride).toList()); } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index d51526a4ed4..fb6d7fb70ec 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -65,6 +65,7 @@ public final class SchemaInfo extends Derived implements SchemaInfoConfig.Produc @Override public void getConfig(SchemaInfoConfig.Builder builder) { + // Append var schemaBuilder = new SchemaInfoConfig.Schema.Builder(); schemaBuilder.name(schema.getName()); addFieldsConfig(schemaBuilder); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java index 7b96f5ab654..51c648aa1be 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java @@ -41,11 +41,10 @@ public class Summaries extends Derived implements SummaryConfig.Producer { @Override public void getConfig(SummaryConfig.Builder builder) { + // Replace builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode()); builder.usev8geopositions(useV8GeoPositions); - for (SummaryClass summaryClass : summaries) { - builder.classes(summaryClass.getSummaryClassConfig()); - } + builder.classes(summaries.stream().map(SummaryClass::getSummaryClassConfig).toList()); } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java index f2ad6a3ba2f..b2b2e9d3a14 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java @@ -108,12 +108,9 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { @Override public void getConfig(VsmfieldsConfig.Builder vsB) { - for (StreamingField streamingField : fields.values()) { - vsB.fieldspec(streamingField.getFieldSpecConfig()); - } - for (StreamingDocumentType streamingDocType : doctypes.values()) { - vsB.documenttype(streamingDocType.getDocTypeConfig()); - } + // Replace + vsB.fieldspec(fields.values().stream().map(StreamingField::getFieldSpecConfig).toList()); + vsB.documenttype(doctypes.values().stream().map(StreamingDocumentType::getDocTypeConfig).toList()); } private static boolean isAttributeField(ImmutableSDField field, boolean isStructField, boolean ignoreAttributeAspect) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java index e3d7becd86a..248cca33bc6 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java @@ -96,14 +96,14 @@ public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { @Override public void getConfig(VsmsummaryConfig.Builder vB) { - for (Map.Entry<SummaryField, List<String>> entry : summaryMap.entrySet()) { - VsmsummaryConfig.Fieldmap.Builder fmB = new VsmsummaryConfig.Fieldmap.Builder().summary(entry.getKey().getName()); - for (String field : entry.getValue()) { - fmB.document(new VsmsummaryConfig.Fieldmap.Document.Builder().field(field)); - } - fmB.command(VsmsummaryConfig.Fieldmap.Command.Enum.valueOf(entry.getKey().getVsmCommand().toString())); - vB.fieldmap(fmB); - } + // Replace + vB.fieldmap( + summaryMap.entrySet().stream().map(entry -> new VsmsummaryConfig.Fieldmap.Builder() + .summary(entry.getKey().getName()) + .document(entry.getValue().stream().map(field -> new VsmsummaryConfig.Fieldmap.Document.Builder().field(field)).toList()) + .command(VsmsummaryConfig.Fieldmap.Command.Enum.valueOf(entry.getKey().getVsmCommand().toString())) + ).toList() + ); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 973ebc8c602..53f72a53f0c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -241,7 +241,7 @@ public class VespaDomBuilder extends VespaModelBuilder { */ private static int getXmlIntegerAttribute(Element spec, String attributeName) { String value = (spec == null) ? null : spec.getAttribute(attributeName); - if (value == null || value.equals("")) { + if (value == null || value.isEmpty()) { return 0; } else { try { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index f58928354db..035ae4a06f5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -173,12 +173,11 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> public void getConfig(QrSearchersConfig.Builder builder) { for (int i = 0; i < searchClusters.size(); i++) { SearchCluster sys = findClusterWithId(searchClusters, i); - QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder(). - name(sys.getClusterName()); + var scB = new QrSearchersConfig.Searchcluster.Builder().name(sys.getClusterName()); for (SchemaInfo spec : sys.schemas().values()) { scB.searchdef(spec.fullSchema().getName()); } - scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); + scB.rankprofiles_configid(sys.getConfigId()); scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); if ( ! (sys instanceof IndexedSearchCluster)) { scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder(). diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 8eca29215d4..c71dbb158b0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -425,7 +425,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addComponent(accessLogComponent); }); } - if (components.size() > 0) { + if ( ! components.isEmpty()) { cluster.removeSimpleComponent(VoidRequestLog.class); cluster.addSimpleComponent(AccessLog.class); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java index 32ac5d0f4ed..334f8d4314a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java @@ -61,18 +61,25 @@ public class DocumentDatabase extends AnyConfigProducer implements public DerivedConfiguration getDerivedConfiguration() { return derivedCfg; } + // These methods will append to the config @Override public void getConfig(IndexInfoConfig.Builder builder) { derivedCfg.getIndexInfo().getConfig(builder); } @Override public void getConfig(IlscriptsConfig.Builder builder) { derivedCfg.getIndexingScript().getConfig(builder); } + @Override public void getConfig(SchemaInfoConfig.Builder builder) { derivedCfg.getSchemaInfo().getConfig(builder); } + + // These methods append as multiple databases join config => TODO will loose information - not good @Override public void getConfig(AttributesConfig.Builder builder) { derivedCfg.getConfig(builder); } @Override public void getConfig(RankProfilesConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } + + // These methods append, TODO unknown usage and consequences @Override public void getConfig(RankingExpressionsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } @Override public void getConfig(RankingConstantsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } @Override public void getConfig(OnnxModelsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); } + + // Below methods will replace config completely @Override public void getConfig(IndexschemaConfig.Builder builder) { derivedCfg.getIndexSchema().getConfig(builder); } @Override public void getConfig(JuniperrcConfig.Builder builder) { derivedCfg.getJuniperrc().getConfig(builder); } @Override public void getConfig(SummaryConfig.Builder builder) { derivedCfg.getSummaries().getConfig(builder); } @Override public void getConfig(ImportedFieldsConfig.Builder builder) { derivedCfg.getImportedFields().getConfig(builder); } - @Override public void getConfig(SchemaInfoConfig.Builder builder) { derivedCfg.getSchemaInfo().getConfig(builder); } @Override public void getConfig(VsmsummaryConfig.Builder builder) { derivedCfg.getVsmSummary().getConfig(builder); } @Override public void getConfig(VsmfieldsConfig.Builder builder) { derivedCfg.getVsmFields().getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 732b4ba0637..920ec4f35fc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -60,6 +60,15 @@ public abstract class SearchCluster extends TreeConfigProducer<AnyConfigProducer return false; } + public String getConfigId(String name) { + for (DocumentDatabase db : documentDbs) { + if (db.getName().equals(name)) { + return db.getConfigId(); + } + } + return ""; + } + /** Returns the schemas that should be active in this cluster. Note: These are added during processing. */ public Map<String, SchemaInfo> schemas() { return Collections.unmodifiableMap(schemas); } |