diff options
author | Arne H Juul <arnej@yahooinc.com> | 2022-03-16 06:19:52 +0000 |
---|---|---|
committer | Arne H Juul <arnej@yahooinc.com> | 2022-03-16 09:29:51 +0000 |
commit | 597e3b4fa3b737c3f1bb8b41517bfa93456b3681 (patch) | |
tree | 955f98bc45423c934fcaef5d8dafe16be04d72b2 /config-model/src/main/java/com/yahoo | |
parent | 7ecbb650f2754adadd5f9b71ea7f43aa8df0425a (diff) |
split per-Schema conversion into its own file
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
2 files changed, 241 insertions, 140 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java new file mode 100644 index 00000000000..c7f476d2c90 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java @@ -0,0 +1,232 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.ReferenceDataType; +import com.yahoo.document.StructDataType; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.searchdefinition.DefaultRankProfile; +import com.yahoo.searchdefinition.DocumentOnlySchema; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.UnrankedRankProfile; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedField; +import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; +import com.yahoo.searchdefinition.parser.ConvertParsedTypes.TypeResolver; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Class converting a collection of schemas from the intermediate format. + * + * @author arnej27959 + **/ +public class ConvertParsedSchemas { + + private final List<ParsedSchema> orderedInput; + private final DocumentTypeManager docMan; + private final ApplicationPackage applicationPackage; + private final FileRegistry fileRegistry; + private final DeployLogger deployLogger; + private final ModelContext.Properties properties; + private final RankProfileRegistry rankProfileRegistry; + private final boolean documentsOnly; + private final ConvertParsedTypes typeConverter; + + // for unit test + ConvertParsedSchemas(List<ParsedSchema> orderedInput, + DocumentTypeManager documentTypeManager) + { + this(orderedInput, documentTypeManager, + MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + new BaseDeployLogger(), + new TestProperties(), + new RankProfileRegistry(), + true); + } + + public ConvertParsedSchemas(List<ParsedSchema> orderedInput, + DocumentTypeManager documentTypeManager, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + boolean documentsOnly) + { + this.orderedInput = orderedInput; + this.docMan = documentTypeManager; + this.applicationPackage = applicationPackage; + this.fileRegistry = fileRegistry; + this.deployLogger = deployLogger; + this.properties = properties; + this.rankProfileRegistry = rankProfileRegistry; + this.documentsOnly = documentsOnly; + this.typeConverter = new ConvertParsedTypes(orderedInput, docMan); + } + + private Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap(); + + public List<Schema> convertToSchemas() { + typeConverter.convert(false); + var resultList = new ArrayList<Schema>(); + for (var parsed : orderedInput) { + Optional<String> inherited; + var inheritList = parsed.getInherited(); + if (inheritList.size() == 0) { + inherited = Optional.empty(); + } else if (inheritList.size() == 1) { + inherited = Optional.of(inheritList.get(0)); + } else { + throw new IllegalArgumentException("schema " + parsed.name() + "cannot inherit more than once"); + } + Schema schema = parsed.getDocumentWithoutSchema() + ? new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties) + : new Schema(parsed.name(), applicationPackage, inherited, fileRegistry, deployLogger, properties); + convertSchema(schema, parsed); + resultList.add(schema); + } + return resultList; + } + + private void convertDocument(Schema schema, ParsedDocument parsed, + ConvertParsedFields fieldConverter) + { + SDDocumentType document = new SDDocumentType(parsed.name()); + for (String inherit : parsed.getInherited()) { + var parent = convertedDocuments.get(inherit); + assert(parent != null); + document.inherit(parent); + } + for (var struct : parsed.getStructs()) { + var structProxy = fieldConverter.convertStructDeclaration(schema, document, struct); + document.addType(structProxy); + } + for (var annotation : parsed.getAnnotations()) { + fieldConverter.convertAnnotation(schema, document, annotation); + } + for (var field : parsed.getFields()) { + var sdf = fieldConverter.convertDocumentField(schema, document, field); + if (field.hasIdOverride()) { + document.setFieldId(sdf, field.idOverride()); + } + } + convertedDocuments.put(parsed.name(), document); + schema.addDocument(document); + } + + private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext) { + var docsum = new DocumentSummary(parsed.name(), schema); + var inheritList = parsed.getInherited(); + if (inheritList.size() == 1) { + docsum.setInherited(inheritList.get(0)); + } else if (inheritList.size() != 0) { + throw new IllegalArgumentException("document-summary "+parsed.name()+" cannot inherit more than once"); + } + if (parsed.getFromDisk()) { + docsum.setFromDisk(true); + } + if (parsed.getOmitSummaryFeatures()) { + docsum.setOmitSummaryFeatures(true); + } + for (var parsedField : parsed.getSummaryFields()) { + DataType dataType = typeContext.resolveType(parsedField.getType()); + var summaryField = new SummaryField(parsedField.name(), dataType); + // XXX does not belong here: + summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); + ConvertParsedFields.convertSummaryFieldSettings(summaryField, parsedField); + docsum.add(summaryField); + } + schema.addSummary(docsum); + } + + private void convertImportField(Schema schema, ParsedSchema.ImportedField f) { + // needs rethinking + var importedFields = schema.temporaryImportedFields().get(); + if (importedFields.hasField(f.asFieldName)) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', import field as '" + f.asFieldName + + "': Field already imported"); + } + importedFields.add(new TemporaryImportedField(f.asFieldName, f.refFieldName, f.foreignFieldName)); + } + + private void convertFieldSet(Schema schema, ParsedFieldSet parsed) { + String setName = parsed.name(); + for (String field : parsed.getFieldNames()) { + schema.fieldSets().addUserFieldSetItem(setName, field); + } + for (String command : parsed.getQueryCommands()) { + schema.fieldSets().userFieldSets().get(setName).queryCommands().add(command); + } + if (parsed.getMatchSettings().isPresent()) { + // same ugliness as SDParser.jj used to have: + var tmp = new SDField(setName, DataType.STRING); + ConvertParsedFields.convertMatchSettings(tmp, parsed.matchSettings()); + schema.fieldSets().userFieldSets().get(setName).setMatching(tmp.getMatching()); + } + } + + private void convertSchema(Schema schema, ParsedSchema parsed) { + if (parsed.hasStemming()) { + schema.setStemming(parsed.getStemming()); + } + parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); + var typeContext = typeConverter.makeContext(parsed.getDocument()); + var fieldConverter = new ConvertParsedFields(typeContext); + convertDocument(schema, parsed.getDocument(), fieldConverter); + for (var field : parsed.getFields()) { + fieldConverter.convertExtraField(schema, field); + } + for (var index : parsed.getIndexes()) { + fieldConverter.convertExtraIndex(schema, index); + } + for (var docsum : parsed.getDocumentSummaries()) { + convertDocumentSummary(schema, docsum, typeContext); + } + for (var importedField : parsed.getImportedFields()) { + convertImportField(schema, importedField); + } + for (var fieldSet : parsed.getFieldSets()) { + convertFieldSet(schema, fieldSet); + } + if (documentsOnly) { + return; // skip ranking-only content, not used for document type generation + } + for (var rankingConstant : parsed.getRankingConstants()) { + schema.rankingConstants().add(rankingConstant); + } + for (var onnxModel : parsed.getOnnxModels()) { + schema.onnxModels().add(onnxModel); + } + rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + var rankConverter = new ConvertParsedRanking(rankProfileRegistry); + for (var rankProfile : parsed.getRankProfiles()) { + rankConverter.convertRankProfile(schema, rankProfile); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index 316170893ee..8ee4a237258 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -124,147 +124,16 @@ public class ConvertSchemaCollection { typeConverter.convert(true); } - private Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap(); - public List<Schema> convertToSchemas() { - typeConverter = new ConvertParsedTypes(orderedInput, docMan); - typeConverter.convert(false); - var resultList = new ArrayList<Schema>(); - for (var parsed : orderedInput) { - Optional<String> inherited; - var inheritList = parsed.getInherited(); - if (inheritList.size() == 0) { - inherited = Optional.empty(); - } else if (inheritList.size() == 1) { - inherited = Optional.of(inheritList.get(0)); - } else { - throw new IllegalArgumentException("schema " + parsed.name() + "cannot inherit more than once"); - } - Schema schema = parsed.getDocumentWithoutSchema() - ? new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties) - : new Schema(parsed.name(), applicationPackage, inherited, fileRegistry, deployLogger, properties); - convertSchema(schema, parsed); - resultList.add(schema); - } - return resultList; - } - - private void convertDocument(Schema schema, ParsedDocument parsed, - ConvertParsedFields fieldConverter) - { - SDDocumentType document = new SDDocumentType(parsed.name()); - for (String inherit : parsed.getInherited()) { - var parent = convertedDocuments.get(inherit); - assert(parent != null); - document.inherit(parent); - } - for (var struct : parsed.getStructs()) { - var structProxy = fieldConverter.convertStructDeclaration(schema, document, struct); - document.addType(structProxy); - } - for (var annotation : parsed.getAnnotations()) { - fieldConverter.convertAnnotation(schema, document, annotation); - } - for (var field : parsed.getFields()) { - var sdf = fieldConverter.convertDocumentField(schema, document, field); - if (field.hasIdOverride()) { - document.setFieldId(sdf, field.idOverride()); - } - } - convertedDocuments.put(parsed.name(), document); - schema.addDocument(document); - } - - private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext) { - var docsum = new DocumentSummary(parsed.name(), schema); - var inheritList = parsed.getInherited(); - if (inheritList.size() == 1) { - docsum.setInherited(inheritList.get(0)); - } else if (inheritList.size() != 0) { - throw new IllegalArgumentException("document-summary "+parsed.name()+" cannot inherit more than once"); - } - if (parsed.getFromDisk()) { - docsum.setFromDisk(true); - } - if (parsed.getOmitSummaryFeatures()) { - docsum.setOmitSummaryFeatures(true); - } - for (var parsedField : parsed.getSummaryFields()) { - DataType dataType = typeContext.resolveType(parsedField.getType()); - var summaryField = new SummaryField(parsedField.name(), dataType); - // XXX does not belong here: - summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); - ConvertParsedFields.convertSummaryFieldSettings(summaryField, parsedField); - docsum.add(summaryField); - } - schema.addSummary(docsum); - } - - private void convertImportField(Schema schema, ParsedSchema.ImportedField f) { - // needs rethinking - var importedFields = schema.temporaryImportedFields().get(); - if (importedFields.hasField(f.asFieldName)) { - throw new IllegalArgumentException("For schema '" + schema.getName() + - "', import field as '" + f.asFieldName + - "': Field already imported"); - } - importedFields.add(new TemporaryImportedField(f.asFieldName, f.refFieldName, f.foreignFieldName)); - } - - private void convertFieldSet(Schema schema, ParsedFieldSet parsed) { - String setName = parsed.name(); - for (String field : parsed.getFieldNames()) { - schema.fieldSets().addUserFieldSetItem(setName, field); - } - for (String command : parsed.getQueryCommands()) { - schema.fieldSets().userFieldSets().get(setName).queryCommands().add(command); - } - if (parsed.getMatchSettings().isPresent()) { - // same ugliness as SDParser.jj used to have: - var tmp = new SDField(setName, DataType.STRING); - ConvertParsedFields.convertMatchSettings(tmp, parsed.matchSettings()); - schema.fieldSets().userFieldSets().get(setName).setMatching(tmp.getMatching()); - } - } - - private void convertSchema(Schema schema, ParsedSchema parsed) { - if (parsed.hasStemming()) { - schema.setStemming(parsed.getStemming()); - } - parsed.getRawAsBase64().ifPresent(value -> schema.enableRawAsBase64(value)); - var typeContext = typeConverter.makeContext(parsed.getDocument()); - var fieldConverter = new ConvertParsedFields(typeContext); - convertDocument(schema, parsed.getDocument(), fieldConverter); - for (var field : parsed.getFields()) { - fieldConverter.convertExtraField(schema, field); - } - for (var index : parsed.getIndexes()) { - fieldConverter.convertExtraIndex(schema, index); - } - for (var docsum : parsed.getDocumentSummaries()) { - convertDocumentSummary(schema, docsum, typeContext); - } - for (var importedField : parsed.getImportedFields()) { - convertImportField(schema, importedField); - } - for (var fieldSet : parsed.getFieldSets()) { - convertFieldSet(schema, fieldSet); - } - if (documentsOnly) { - return; // skip ranking-only content, not used for document type generation - } - for (var rankingConstant : parsed.getRankingConstants()) { - schema.rankingConstants().add(rankingConstant); - } - for (var onnxModel : parsed.getOnnxModels()) { - schema.onnxModels().add(onnxModel); - } - rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); - rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); - var rankConverter = new ConvertParsedRanking(rankProfileRegistry); - for (var rankProfile : parsed.getRankProfiles()) { - rankConverter.convertRankProfile(schema, rankProfile); - } + var converter = new ConvertParsedSchemas(orderedInput, + docMan, + applicationPackage, + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + documentsOnly); + return converter.convertToSchemas(); } } |