diff options
Diffstat (limited to 'config-model/src/main/java/com/yahoo')
5 files changed, 296 insertions, 24 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 51f3455762c..480b6590555 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -79,6 +79,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useV8GeoPositions = false; private List<String> environmentVariables = List.of(); private boolean avoidRenamingSummaryFeatures = false; + private boolean experimentalSdParsing = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -138,6 +139,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } @Override public boolean avoidRenamingSummaryFeatures() { return this.avoidRenamingSummaryFeatures; } + @Override public boolean experimentalSdParsing() { return this.experimentalSdParsing; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -375,6 +377,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setExperimentalSdParsing(boolean value) { + this.experimentalSdParsing = value; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java index 64688a7e70d..16eef798acd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java @@ -84,6 +84,7 @@ public class Application { List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas); for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { + new FieldOperationApplierForStructs().processSchemaFields(schema); new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? new Processing(properties).process(schema, logger, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java index 533546b4d39..708a2b932c6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java @@ -15,6 +15,9 @@ import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; +import com.yahoo.searchdefinition.parser.ConvertSchemaCollection; +import com.yahoo.searchdefinition.parser.IntermediateCollection; +import com.yahoo.searchdefinition.parser.IntermediateParser; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.parser.SDParser; import com.yahoo.searchdefinition.parser.SimpleCharStream; @@ -43,6 +46,7 @@ import java.util.Set; */ public class ApplicationBuilder { + private final IntermediateCollection mediator; private final ApplicationPackage applicationPackage; private final List<Schema> schemas = new ArrayList<>(); private final DocumentTypeManager documentTypeManager = new DocumentTypeManager(); @@ -118,6 +122,7 @@ public class ApplicationBuilder { RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, boolean documentsOnly) { + this.mediator = new IntermediateCollection(deployLogger, properties); this.applicationPackage = applicationPackage; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; @@ -133,13 +138,17 @@ public class ApplicationBuilder { * Adds a schema to this application. * * @param fileName the name of the file to import - * @return the name of the imported object * @throws IOException thrown if the file can not be read for some reason * @throws ParseException thrown if the file does not contain a valid search definition */ - public Schema addSchemaFile(String fileName) throws IOException, ParseException { + public void addSchemaFile(String fileName) throws IOException, ParseException { + if (properties.featureFlags().experimentalSdParsing()) { + var parsedName = mediator.addSchemaFromFile(fileName); + addRankProfileFiles(parsedName); + return; + } File file = new File(fileName); - return addSchema(IOUtils.readFile(file)); + addSchema(IOUtils.readFile(file)); } /** @@ -149,8 +158,19 @@ public class ApplicationBuilder { * @param reader the reader whose content to import */ public void addSchema(NamedReader reader) { + if (properties.featureFlags().experimentalSdParsing()) { + try { + var parsedName = mediator.addSchemaFromReader(reader); + addRankProfileFiles(parsedName); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); + } + return; + } try { - String schemaName = addSchema(IOUtils.readAll(reader)).getName(); + Schema schema = createSchema(IOUtils.readAll(reader)); + add(schema); + String schemaName = schema.getName(); String schemaFileName = stripSuffix(reader.getName(), ApplicationPackage.SD_NAME_SUFFIX); if ( ! schemaFileName.equals(schemaName)) { throw new IllegalArgumentException("The file containing schema '" + schemaName + "' must be named '" + @@ -176,8 +196,13 @@ public class ApplicationBuilder { * * @param schemaString the content of the schema */ - public Schema addSchema(String schemaString) throws ParseException { - return add(createSchema(schemaString)); + public void addSchema(String schemaString) throws ParseException { + if (properties.featureFlags().experimentalSdParsing()) { + var parsed = mediator.addSchemaFromString(schemaString); + addRankProfileFiles(parsed.name()); + return; + } + add(createSchema(schemaString)); } /** @@ -202,6 +227,9 @@ public class ApplicationBuilder { } private Schema parseSchema(String schemaString) throws ParseException { + if (properties.featureFlags().experimentalSdParsing()) { + throw new IllegalArgumentException("should use new parser only"); + } SimpleCharStream stream = new SimpleCharStream(schemaString); try { return parserOf(stream).schema(documentTypeManager); @@ -215,6 +243,10 @@ public class ApplicationBuilder { private void addRankProfileFiles(Schema schema) { if (applicationPackage == null) return; + if (properties.featureFlags().experimentalSdParsing()) { + throw new IllegalArgumentException("should use new parser only"); + } + Path legacyRankProfilePath = ApplicationPackage.SEARCH_DEFINITIONS_DIR.append(schema.getName()); for (NamedReader reader : applicationPackage.getFiles(legacyRankProfilePath, ".profile")) parseRankProfile(reader, schema); @@ -224,8 +256,28 @@ public class ApplicationBuilder { parseRankProfile(reader, schema); } + private void addRankProfileFiles(String schemaName) throws ParseException { + if (applicationPackage == null) return; + if (! properties.featureFlags().experimentalSdParsing()) { + throw new IllegalArgumentException("should use old parser only"); + } + + Path legacyRankProfilePath = ApplicationPackage.SEARCH_DEFINITIONS_DIR.append(schemaName); + for (NamedReader reader : applicationPackage.getFiles(legacyRankProfilePath, ".profile")) { + mediator.addRankProfileFile(schemaName, reader); + } + + Path rankProfilePath = ApplicationPackage.SCHEMAS_DIR.append(schemaName); + for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile")) { + mediator.addRankProfileFile(schemaName, reader); + } + } + /** Parses the rank profile of the given reader and adds it to the rank profile registry for this schema. */ private void parseRankProfile(NamedReader reader, Schema schema) { + if (properties.featureFlags().experimentalSdParsing()) { + throw new IllegalArgumentException("should use new parser only"); + } try { SimpleCharStream stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); try { @@ -256,7 +308,19 @@ public class ApplicationBuilder { */ public Application build(boolean validate) { if (application != null) throw new IllegalStateException("Application already built"); - + if (properties.featureFlags().experimentalSdParsing()) { + var converter = new ConvertSchemaCollection(mediator, + documentTypeManager, + applicationPackage, + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + documentsOnly); + for (var schema : converter.convertToSchemas()) { + add(schema); + } + } application = new Application(applicationPackage, schemas, rankProfileRegistry, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 16bf37902f5..5e5623e2319 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -35,17 +35,32 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier { Iterator<Field> fields = anyType.fieldIterator(); while (fields.hasNext()) { SDField field = (SDField) fields.next(); - DataType structUsedByField = field.getFirstStructRecursive(); - if (structUsedByField == null) { - continue; - } - if (structUsedByField.getName().equals(structType.getName())) { - //this field is using this type!! - field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); - field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching()); - } + maybePopulateField(sdoc, field, structType); } } } + private void maybePopulateField(SDDocumentType sdoc, SDField field, SDDocumentType structType) { + DataType structUsedByField = field.getFirstStructRecursive(); + if (structUsedByField == null) { + return; + } + if (structUsedByField.getName().equals(structType.getName())) { + //this field is using this type!! + field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); + field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching()); + } + } + + public void processSchemaFields(Schema schema) { + var sdoc = schema.getDocument(); + if (sdoc == null) return; + for (SDDocumentType type : sdoc.getAllTypes()) { + if (type.isStruct()) { + for (SDField field : schema.allExtraFields()) { + maybePopulateField(sdoc, field, type); + } + } + } + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index fb0003ca4f9..1dc10c57eb5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -1,25 +1,42 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.parser; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.PositionDataType; import com.yahoo.document.ReferenceDataType; import com.yahoo.document.StructDataType; -import com.yahoo.document.PositionDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.searchdefinition.DefaultRankProfile; +import com.yahoo.searchdefinition.DocumentOnlySchema; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.UnrankedRankProfile; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.TemporaryImportedField; +import com.yahoo.searchdefinition.parser.ConvertParsedTypes.TypeResolver; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.Optional; /** * Class converting a collection of schemas from the intermediate format. - * For now only conversion to DocumentType (with contents). * * @author arnej27959 **/ @@ -28,12 +45,45 @@ public class ConvertSchemaCollection { private final IntermediateCollection input; private final List<ParsedSchema> orderedInput = new ArrayList<>(); private final DocumentTypeManager docMan; + private final ApplicationPackage applicationPackage; + private final FileRegistry fileRegistry; + private final DeployLogger deployLogger; + private final ModelContext.Properties properties; + private final RankProfileRegistry rankProfileRegistry; + private final boolean documentsOnly; + + // for unit test + ConvertSchemaCollection(IntermediateCollection input, + DocumentTypeManager documentTypeManager) + { + this(input, documentTypeManager, + MockApplicationPackage.createEmpty(), + new MockFileRegistry(), + new BaseDeployLogger(), + new TestProperties(), + new RankProfileRegistry(), + true); + } public ConvertSchemaCollection(IntermediateCollection input, - DocumentTypeManager documentTypeManager) + DocumentTypeManager documentTypeManager, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + boolean documentsOnly) { this.input = input; this.docMan = documentTypeManager; + this.applicationPackage = applicationPackage; + this.fileRegistry = fileRegistry; + this.deployLogger = deployLogger; + this.properties = properties; + this.rankProfileRegistry = rankProfileRegistry; + this.documentsOnly = documentsOnly; + + input.resolveInternalConnections(); order(); pushTypesToDocuments(); } @@ -64,8 +114,143 @@ public class ConvertSchemaCollection { } } + private ConvertParsedTypes typeConverter; + public void convertTypes() { - var converter = new ConvertParsedTypes(orderedInput, docMan); - converter.convert(); + if (typeConverter == null) { + typeConverter = new ConvertParsedTypes(orderedInput, docMan); + typeConverter.convert(); + } + } + + public List<Schema> convertToSchemas() { + convertTypes(); + var resultList = new ArrayList<Schema>(); + for (var parsed : orderedInput) { + Optional<String> inherited; + var inheritList = parsed.getInherited(); + if (inheritList.size() == 0) { + inherited = Optional.empty(); + } else if (inheritList.size() == 1) { + inherited = Optional.of(inheritList.get(0)); + } else { + throw new IllegalArgumentException("schema " + parsed.name() + "cannot inherit more than once"); + } + Schema schema = parsed.getDocumentWithoutSchema() + ? new DocumentOnlySchema(applicationPackage, fileRegistry, deployLogger, properties) + : new Schema(parsed.name(), applicationPackage, inherited, fileRegistry, deployLogger, properties); + convertSchema(schema, parsed); + resultList.add(schema); + } + return resultList; } + + private void convertDocument(Schema schema, ParsedDocument parsed, + ConvertParsedFields fieldConverter) + { + SDDocumentType document = new SDDocumentType(parsed.name()); + for (String inherit : parsed.getInherited()) { + document.inherit(new DataTypeName(inherit)); + } + for (var struct : parsed.getStructs()) { + fieldConverter.convertStructDeclaration(schema, document, struct); + } + for (var field : parsed.getFields()) { + var sdf = fieldConverter.convertDocumentField(schema, document, field); + if (field.hasIdOverride()) { + document.setFieldId(sdf, field.idOverride()); + } + } + schema.addDocument(document); + } + + private void convertDocumentSummary(Schema schema, ParsedDocumentSummary parsed, TypeResolver typeContext) { + var docsum = new DocumentSummary(parsed.name(), schema); + var inheritList = parsed.getInherited(); + if (inheritList.size() == 1) { + docsum.setInherited(inheritList.get(0)); + } else if (inheritList.size() != 0) { + throw new IllegalArgumentException("document-summary "+parsed.name()+" cannot inherit more than once"); + } + if (parsed.getFromDisk()) { + docsum.setFromDisk(true); + } + if (parsed.getOmitSummaryFeatures()) { + docsum.setOmitSummaryFeatures(true); + } + for (var parsedField : parsed.getSummaryFields()) { + DataType dataType = typeContext.resolveType(parsedField.getType()); + var summaryField = new SummaryField(parsedField.name(), dataType); + // XXX does not belong here: + summaryField.setVsmCommand(SummaryField.VsmCommand.FLATTENSPACE); + ConvertParsedFields.convertSummaryFieldSettings(summaryField, parsedField); + docsum.add(summaryField); + } + schema.addSummary(docsum); + } + + private void convertImportField(Schema schema, ParsedSchema.ImportedField f) { + // needs rethinking + var importedFields = schema.temporaryImportedFields().get(); + if (importedFields.hasField(f.asFieldName)) { + throw new IllegalArgumentException("For schema '" + schema.getName() + + "', import field as '" + f.asFieldName + + "': Field already imported"); + } + importedFields.add(new TemporaryImportedField(f.asFieldName, f.refFieldName, f.foreignFieldName)); + } + + private void convertFieldSet(Schema schema, ParsedFieldSet parsed) { + String setName = parsed.name(); + for (String field : parsed.getFieldNames()) { + schema.fieldSets().addUserFieldSetItem(setName, field); + } + for (String command : parsed.getQueryCommands()) { + schema.fieldSets().userFieldSets().get(setName).queryCommands().add(command); + } + if (parsed.getMatchSettings().isPresent()) { + // same ugliness as SDParser.jj used to have: + var tmp = new SDField(setName, DataType.STRING); + ConvertParsedFields.convertMatchSettings(tmp, parsed.matchSettings()); + schema.fieldSets().userFieldSets().get(setName).setMatching(tmp.getMatching()); + } + } + + private void convertSchema(Schema schema, ParsedSchema parsed) { + if (parsed.hasStemming()) { + schema.setStemming(parsed.getStemming()); + } + schema.enableRawAsBase64(parsed.getRawAsBase64()); + var typeContext = typeConverter.makeContext(parsed.getDocument()); + var fieldConverter = new ConvertParsedFields(typeContext); + convertDocument(schema, parsed.getDocument(), fieldConverter); + for (var field : parsed.getFields()) { + fieldConverter.convertExtraField(schema, field); + } + for (var index : parsed.getIndexes()) { + fieldConverter.convertExtraIndex(schema, index); + } + for (var docsum : parsed.getDocumentSummaries()) { + convertDocumentSummary(schema, docsum, typeContext); + } + for (var importedField : parsed.getImportedFields()) { + convertImportField(schema, importedField); + } + for (var fieldSet : parsed.getFieldSets()) { + convertFieldSet(schema, fieldSet); + } + for (var rankingConstant : parsed.getRankingConstants()) { + schema.rankingConstants().add(rankingConstant); + } + for (var onnxModel : parsed.getOnnxModels()) { + schema.onnxModels().add(onnxModel); + } + rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + var rankConverter = new ConvertParsedRanking(rankProfileRegistry); + for (var rankProfile : parsed.getRankProfiles()) { + rankConverter.convertRankProfile(schema, rankProfile); + } + } + } |