diff options
author | Jon Bratseth <bratseth@oath.com> | 2022-01-26 13:50:16 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-26 13:50:16 +0100 |
commit | aef6dbaa7a3c4d885485b0905b9c2f84d0aedc6f (patch) | |
tree | ecfb2f612bf7b183d70ee273e0a83bf0dbc5db10 /config-model/src/main/java/com/yahoo/searchdefinition | |
parent | 1ecbb8ced4c2599155a2438625987b78e174bd5a (diff) | |
parent | 5869796c08003f41f0ac8e4738414d136b075b1d (diff) |
Merge pull request #20938 from vespa-engine/bratseth/modular-profiles
Bratseth/modular profiles
Diffstat (limited to 'config-model/src/main/java/com/yahoo/searchdefinition')
19 files changed, 435 insertions, 350 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java index fb76c945344..2dda670f07c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Application.java @@ -3,10 +3,20 @@ package com.yahoo.searchdefinition; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.searchdefinition.derived.SearchOrderer; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.searchdefinition.processing.Processor; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Set; /** * A collection of objects representing the content of an application package. @@ -18,28 +28,79 @@ import java.util.Map; public class Application { private final ApplicationPackage applicationPackage; - private final Map<String, Schema> schemas = new LinkedHashMap<>(); + private final Map<String, Schema> schemas; + private final DocumentModel documentModel; - public Application(ApplicationPackage applicationPackage) { + public Application(ApplicationPackage applicationPackage, + List<Schema> schemas, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles, + ModelContext.Properties properties, + boolean documentsOnly, + boolean validate, + Set<Class<? extends Processor>> processorsToSkip, + DeployLogger logger) { this.applicationPackage = applicationPackage; - } - public ApplicationPackage applicationPackage() { return applicationPackage; } + Map<String, Schema> schemaMap = new LinkedHashMap<>(); + for (Schema schema : schemas) { + if (schemaMap.containsKey(schema.getName())) + throw new IllegalArgumentException("Duplicate schema '" + schema.getName() + "' in " + this); + schemaMap.put(schema.getName(), schema); + } + this.schemas = Collections.unmodifiableMap(schemaMap); + + schemas.forEach(schema -> schema.setOwner(this)); + if (validate) + schemas.forEach(schema -> schema.validate(logger)); + + new TemporarySDTypeResolver(schemas, logger).process(); + + List<SDDocumentType> sdocs = new ArrayList<>(); + sdocs.add(SDDocumentType.VESPA_DOCUMENT); + for (Schema schema : schemas) { + if (schema.hasDocument()) { + sdocs.add(schema.getDocument()); + } + } + + var orderer = new SDDocumentTypeOrderer(sdocs, logger); + orderer.process(); + for (SDDocumentType sdoc : orderer.getOrdered()) { + new FieldOperationApplierForStructs().process(sdoc); + new FieldOperationApplier().process(sdoc); + } + var resolver = new DocumentReferenceResolver(schemas); + sdocs.forEach(resolver::resolveReferences); + sdocs.forEach(resolver::resolveInheritedReferences); + var importedFieldsEnumerator = new ImportedFieldsEnumerator(schemas); + sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); - public void add(Schema schema) { - if (schemas.containsKey(schema.getName())) - throw new IllegalArgumentException("Duplicate schema '" + schema.getName() + "' in " + this); - schemas.put(schema.getName(), schema); + if (validate) + new DocumentGraphValidator().validateDocumentGraph(sdocs); + + List<Schema> schemasSomewhatOrdered = new ArrayList<>(schemas); + for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { + new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? + new Processing(properties).process(schema, + logger, + rankProfileRegistry, + queryProfiles, + validate, + documentsOnly, + processorsToSkip); + } + + this.documentModel = new DocumentModelBuilder().build(schemasSomewhatOrdered); } + public ApplicationPackage applicationPackage() { return applicationPackage; } + /** Returns an unmodifiable list of the schemas of this application */ - public Map<String, Schema> schemas() { return Collections.unmodifiableMap(schemas); } + public Map<String, Schema> schemas() { return schemas; } - /** Validates this. Must be called after all content is added to it. */ - public void validate(DeployLogger logger) { - schemas.values().forEach(schema -> schema.validate(logger)); - } + public DocumentModel documentModel() { return documentModel; } @Override public String toString() { return "application " + applicationPackage.getApplicationId(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SchemaBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java index 51466a5dbfa..ac4ac649a28 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SchemaBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java @@ -12,15 +12,13 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DocumentTypeManager; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; +import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; -import com.yahoo.searchdefinition.derived.SearchOrderer; -import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.searchdefinition.parser.SDParser; import com.yahoo.searchdefinition.parser.SimpleCharStream; import com.yahoo.searchdefinition.parser.TokenMgrException; -import com.yahoo.searchdefinition.processing.Processing; import com.yahoo.searchdefinition.processing.Processor; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -28,28 +26,26 @@ import com.yahoo.yolean.Exceptions; import java.io.File; import java.io.IOException; +import java.io.Reader; import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Set; /** - * Helper class for building {@link Schema}s. The pattern for using this is to 1) Import - * all available search definitions, using the importXXX() methods, 2) provide the available rank types and rank - * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built - * search objects using the {@link #getSchema(String)} method. + * Application builder. Usage: + * 1) Add all schemas, using the addXXX() methods, + * 2) provide the available rank types and rank expressions, using the setRankXXX() methods, + * 3) invoke the {@link #build} method */ -// NOTE: Since this was created we have added Application, and much of the content in this should migrate there. -public class SchemaBuilder { +public class ApplicationBuilder { - private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); - private final DocumentModel model = new DocumentModel(); - private final Application application; + private final ApplicationPackage applicationPackage; + private final List<Schema> schemas = new ArrayList<>(); + private final DocumentTypeManager documentTypeManager = new DocumentTypeManager(); private final RankProfileRegistry rankProfileRegistry; private final QueryProfileRegistry queryProfileRegistry; private final FileRegistry fileRegistry; @@ -58,221 +54,230 @@ public class SchemaBuilder { /** True to build the document aspect only, skipping instantiation of rank profiles */ private final boolean documentsOnly; - private boolean isBuilt = false; + private Application application; private final Set<Class<? extends Processor>> processorsToSkip = new HashSet<>(); /** For testing only */ - public SchemaBuilder() { + public ApplicationBuilder() { this(new RankProfileRegistry(), new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(DeployLogger deployLogger) { + public ApplicationBuilder(DeployLogger deployLogger) { this(MockApplicationPackage.createEmpty(), deployLogger); } /** For testing only */ - public SchemaBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + public ApplicationBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { this(MockApplicationPackage.createEmpty(), deployLogger, rankProfileRegistry); } /** Used for generating documents for typed access to document fields in Java */ - public SchemaBuilder(boolean documentsOnly) { + public ApplicationBuilder(boolean documentsOnly) { this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); } /** For testing only */ - public SchemaBuilder(ApplicationPackage app, DeployLogger deployLogger) { + public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger) { this(app, new MockFileRegistry(), deployLogger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + public ApplicationBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { this(app, new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(RankProfileRegistry rankProfileRegistry) { + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry) { this(rankProfileRegistry, new QueryProfileRegistry()); } /** For testing only */ - public SchemaBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { this(rankProfileRegistry, queryProfileRegistry, new TestProperties()); } - public SchemaBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { + public ApplicationBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), properties, rankProfileRegistry, queryProfileRegistry); } - public SchemaBuilder(ApplicationPackage app, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { + public ApplicationBuilder(ApplicationPackage app, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { this(app, fileRegistry, deployLogger, properties, rankProfileRegistry, queryProfileRegistry, false); } - private SchemaBuilder(ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry, - boolean documentsOnly) { - this.application = new Application(applicationPackage); + private ApplicationBuilder(ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry, + boolean documentsOnly) { + this.applicationPackage = applicationPackage; this.rankProfileRegistry = rankProfileRegistry; this.queryProfileRegistry = queryProfileRegistry; this.fileRegistry = fileRegistry; this.deployLogger = deployLogger; this.properties = properties; this.documentsOnly = documentsOnly; + for (NamedReader reader : applicationPackage.getSchemas()) + addSchema(reader); } /** - * Import search definition. + * Adds a schema to this application. * * @param fileName the name of the file to import * @return the name of the imported object * @throws IOException thrown if the file can not be read for some reason * @throws ParseException thrown if the file does not contain a valid search definition */ - public String importFile(String fileName) throws IOException, ParseException { + public Schema addSchemaFile(String fileName) throws IOException, ParseException { File file = new File(fileName); - return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent()); - } - - private String importFile(Path file) throws IOException, ParseException { - return importFile(file.toString()); + return addSchema(IOUtils.readFile(file), file.getAbsoluteFile().getParent()); } /** - * Reads and parses the search definition string provided by the given reader. Once all search definitions have been - * imported, call {@link #build()}. + * Reads and parses the schema string provided by the given reader. Once all schemas have been + * imported, call {@link #build}. * - * @param reader the reader whose content to import - * @param searchDefDir the path to use when resolving file references - * @return the name of the imported object - * @throws ParseException thrown if the file does not contain a valid search definition + * @param reader the reader whose content to import */ - public String importReader(NamedReader reader, String searchDefDir) throws IOException, ParseException { - return importString(IOUtils.readAll(reader), searchDefDir); + private void addSchema(NamedReader reader) { + try { + String schemaName = addSchema(IOUtils.readAll(reader), reader.getName()).getName(); + String schemaFileName = stripSuffix(reader.getName(), ApplicationPackage.SD_NAME_SUFFIX); + if ( ! schemaFileName.equals(schemaName)) { + throw new IllegalArgumentException("The file containing schema '" + schemaName + "' must be named '" + + schemaName + ApplicationPackage.SD_NAME_SUFFIX + "', not " + reader.getName()); + } + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse schema file '" + reader.getName() + "'", e); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read schema file '" + reader.getName() + "'", e); + } finally { + closeIgnoreException(reader.getReader()); + } + } + + private static String stripSuffix(String readerName, String suffix) { + if ( ! readerName.endsWith(suffix)) + throw new IllegalArgumentException("Schema '" + readerName + "' does not end with " + suffix); + return readerName.substring(0, readerName.length() - suffix.length()); } /** - * Import search definition. + * Adds a schema to this application. * - * @param str the string to parse - * @return the name of the imported object + * @param string the string to parse + * @return the schema * @throws ParseException thrown if the file does not contain a valid search definition */ - public String importString(String str) throws ParseException { - return importString(str, null); + public Schema addSchema(String string) throws ParseException { + return addSchema(string, null); } - private String importString(String str, String searchDefDir) throws ParseException { - SimpleCharStream stream = new SimpleCharStream(str); - try { - return importRawSchema(new SDParser(stream, fileRegistry, deployLogger, properties, application, - rankProfileRegistry, documentsOnly) - .schema(docTypeMgr, searchDefDir)); - } catch (TokenMgrException e) { - throw new ParseException("Unknown symbol: " + e.getMessage()); - } catch (ParseException pe) { - throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); - } + private Schema addSchema(String schemaString, String schemaPath) throws ParseException { + return add(createSchema(schemaString, schemaPath)); } /** - * Registers the given schema to the application to be built during {@link #build()}. A + * Registers the given schema to the application to be built during {@link #build}. A * {@link Schema} object is considered to be "raw" if it has not already been processed. This is the case for most * programmatically constructed schemas used in unit tests. * * @param schema the object to import - * @return the name of the imported object * @throws IllegalArgumentException if the given search object has already been processed */ - public String importRawSchema(Schema schema) { + public Schema add(Schema schema) { if (schema.getName() == null) throw new IllegalArgumentException("Schema has no name"); - String rawName = schema.getName(); - application.add(schema); - return rawName; + schemas.add(schema); + return schema; } - /** - * Processes and finalizes the schemas of this. - * Only for testing. - * - * @throws IllegalStateException Thrown if this method has already been called. - */ - public void build() { - build(true); + private Schema createSchema(String schemaString, String schemaPath) throws ParseException { + Schema schema = parseSchema(schemaString, schemaPath); + addRankProfileFiles(schema, schemaPath); + return schema; } - /** - * Processes and finalizes the schemas of this. - * - * @throws IllegalStateException thrown if this method has already been called - */ - public void build(boolean validate) { - if (isBuilt) throw new IllegalStateException("Application already built"); - - new TemporarySDTypeResolver(application.schemas().values(), deployLogger).process(); + private Schema parseSchema(String schemaString, String schemaPath) throws ParseException { + SimpleCharStream stream = new SimpleCharStream(schemaString); + try { + return new SDParser(stream, applicationPackage, fileRegistry, deployLogger, properties, + rankProfileRegistry, documentsOnly) + .schema(documentTypeManager, schemaPath); + } catch (TokenMgrException e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); + } + } - if (validate) - application.validate(deployLogger); + private void addRankProfileFiles(Schema schema, String schemaPath) { + if (applicationPackage == null || schemaPath == null) return; + Path rankProfilePath = Path.fromString(schemaPath).append(schema.getName()); + System.out.println("Adding external rank profiles for " + schema + ": Have " + + applicationPackage.getFiles(rankProfilePath, ".profile").size() + + " files in " + rankProfilePath); + for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile")) { + parseRankProfile(reader, schema); + } + } - List<SDDocumentType> sdocs = new ArrayList<>(); - sdocs.add(SDDocumentType.VESPA_DOCUMENT); - for (Schema schema : application.schemas().values()) { - if (schema.hasDocument()) { - sdocs.add(schema.getDocument()); + /** Parses the rank profile of the given reader and adds it to the rank profile registry for this schema. */ + private void parseRankProfile(NamedReader reader, Schema schema) { + try { + SimpleCharStream stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); + try { + new SDParser(stream, applicationPackage, fileRegistry, deployLogger, properties, + rankProfileRegistry, documentsOnly) + .rankProfile(schema); + } catch (TokenMgrException e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); } } - - var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); - orderer.process(); - for (SDDocumentType sdoc : orderer.getOrdered()) { - new FieldOperationApplierForStructs().process(sdoc); - new FieldOperationApplier().process(sdoc); + catch (IOException e) { + throw new IllegalArgumentException("Could not read rank profile " + reader.getName(), e); } - - var resolver = new DocumentReferenceResolver(application.schemas().values()); - sdocs.forEach(resolver::resolveReferences); - sdocs.forEach(resolver::resolveInheritedReferences); - var importedFieldsEnumerator = new ImportedFieldsEnumerator(application.schemas().values()); - sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); - - if (validate) - new DocumentGraphValidator().validateDocumentGraph(sdocs); - - var builder = new DocumentModelBuilder(model); - List<Schema> schemasSomewhatOrdered = new ArrayList<>(application.schemas().values()); - for (Schema schema : new SearchOrderer().order(schemasSomewhatOrdered)) { - new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? - process(schema, new QueryProfiles(queryProfileRegistry, deployLogger), validate); + catch (ParseException e) { + throw new IllegalArgumentException("Could not parse rank profile " + reader.getName(), e); } - builder.addToModel(schemasSomewhatOrdered); - isBuilt = true; } - /** Returns a modifiable set of processors we should skip for these schemas. Useful for testing. */ - public Set<Class<? extends Processor>> processorsToSkip() { return processorsToSkip; } + /** - * Processes and returns the given {@link Schema} object. This method has been factored out of the {@link - * #build()} method so that subclasses can choose not to build anything. + * Processes and finalizes the schemas of this. + * + * @throws IllegalStateException thrown if this method has already been called */ - private void process(Schema schema, QueryProfiles queryProfiles, boolean validate) { - new Processing(properties).process(schema, deployLogger, - rankProfileRegistry, queryProfiles, - validate, documentsOnly, - processorsToSkip); + public Application build(boolean validate) { + if (application != null) throw new IllegalStateException("Application already built"); + + application = new Application(applicationPackage, + schemas, + rankProfileRegistry, + new QueryProfiles(queryProfileRegistry, deployLogger), + properties, + documentsOnly, + validate, + processorsToSkip, + deployLogger); + return application; } + /** Returns a modifiable set of processors we should skip for these schemas. Useful for testing. */ + public Set<Class<? extends Processor>> processorsToSkip() { return processorsToSkip; } + /** * Convenience method to call {@link #getSchema(String)} when there is only a single {@link Schema} object * built. This method will never return null. @@ -281,7 +286,7 @@ public class SchemaBuilder { * @throws IllegalStateException if there is not exactly one search. */ public Schema getSchema() { - if ( ! isBuilt) throw new IllegalStateException("Application not built."); + if (application == null) throw new IllegalStateException("Application not built"); if (application.schemas().size() != 1) throw new IllegalStateException("This call only works if we have 1 schema. Schemas: " + application.schemas().values()); @@ -289,9 +294,7 @@ public class SchemaBuilder { return application.schemas().values().stream().findAny().get(); } - public DocumentModel getModel() { - return model; - } + public DocumentModel getModel() { return application.documentModel(); } /** * Returns the built {@link Schema} object that has the given name. If the name is unknown, this method will simply @@ -300,10 +303,10 @@ public class SchemaBuilder { * @param name the name of the schema to return, * or null to return the only one or throw an exception if there are multiple to choose from * @return the built object, or null if none with this name - * @throws IllegalStateException if {@link #build()} has not been called. + * @throws IllegalStateException if {@link #build} has not been called. */ public Schema getSchema(String name) { - if ( ! isBuilt) throw new IllegalStateException("Application not built."); + if (application == null) throw new IllegalStateException("Application not built"); if (name == null) return getSchema(); return application.schemas().get(name); } @@ -323,24 +326,24 @@ public class SchemaBuilder { * Convenience factory method to import and build a {@link Schema} object from a string. * * @param sd the string to build from - * @return the built {@link SchemaBuilder} object + * @return the built {@link ApplicationBuilder} object * @throws ParseException thrown if there is a problem parsing the string */ - public static SchemaBuilder createFromString(String sd) throws ParseException { + public static ApplicationBuilder createFromString(String sd) throws ParseException { return createFromString(sd, new BaseDeployLogger()); } - public static SchemaBuilder createFromString(String sd, DeployLogger logger) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(logger); - builder.importString(sd); + public static ApplicationBuilder createFromString(String sd, DeployLogger logger) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(logger); + builder.addSchema(sd); builder.build(true); return builder; } - public static SchemaBuilder createFromStrings(DeployLogger logger, String ... schemas) throws ParseException { - SchemaBuilder builder = new SchemaBuilder(logger); + public static ApplicationBuilder createFromStrings(DeployLogger logger, String ... schemas) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(logger); for (var schema : schemas) - builder.importString(schema); + builder.addSchema(schema); builder.build(true); return builder; } @@ -349,26 +352,26 @@ public class SchemaBuilder { * Convenience factory method to import and build a {@link Schema} object from a file. Only for testing. * * @param fileName the file to build from - * @return the built {@link SchemaBuilder} object + * @return the built {@link ApplicationBuilder} object * @throws IOException if there was a problem reading the file. * @throws ParseException if there was a problem parsing the file content. */ - public static SchemaBuilder createFromFile(String fileName) throws IOException, ParseException { + public static ApplicationBuilder createFromFile(String fileName) throws IOException, ParseException { return createFromFile(fileName, new BaseDeployLogger()); } /** * Convenience factory methdd to create a SearchBuilder from multiple SD files. Only for testing. */ - public static SchemaBuilder createFromFiles(Collection<String> fileNames) throws IOException, ParseException { + public static ApplicationBuilder createFromFiles(Collection<String> fileNames) throws IOException, ParseException { return createFromFiles(fileNames, new BaseDeployLogger()); } - public static SchemaBuilder createFromFile(String fileName, DeployLogger logger) throws IOException, ParseException { + public static ApplicationBuilder createFromFile(String fileName, DeployLogger logger) throws IOException, ParseException { return createFromFile(fileName, logger, new RankProfileRegistry(), new QueryProfileRegistry()); } - private static SchemaBuilder createFromFiles(Collection<String> fileNames, DeployLogger logger) throws IOException, ParseException { + private static ApplicationBuilder createFromFiles(Collection<String> fileNames, DeployLogger logger) throws IOException, ParseException { return createFromFiles(fileNames, new MockFileRegistry(), logger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); } @@ -378,14 +381,14 @@ public class SchemaBuilder { * @param fileName the file to build from. * @param deployLogger logger for deploy messages. * @param rankProfileRegistry registry for rank profiles. - * @return the built {@link SchemaBuilder} object. + * @return the built {@link ApplicationBuilder} object. * @throws IOException if there was a problem reading the file. * @throws ParseException if there was a problem parsing the file content. */ - private static SchemaBuilder createFromFile(String fileName, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) + private static ApplicationBuilder createFromFile(String fileName, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { return createFromFiles(Collections.singletonList(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, queryprofileRegistry); @@ -394,62 +397,62 @@ public class SchemaBuilder { /** * Convenience factory methdd to create a SearchBuilder from multiple SD files.. */ - private static SchemaBuilder createFromFiles(Collection<String> fileNames, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) + private static ApplicationBuilder createFromFiles(Collection<String> fileNames, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryprofileRegistry) throws IOException, ParseException { - SchemaBuilder builder = new SchemaBuilder(MockApplicationPackage.createEmpty(), - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryprofileRegistry); + ApplicationBuilder builder = new ApplicationBuilder(MockApplicationPackage.createEmpty(), + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + queryprofileRegistry); for (String fileName : fileNames) { - builder.importFile(fileName); + builder.addSchemaFile(fileName); } builder.build(true); return builder; } - public static SchemaBuilder createFromDirectory(String dir, FileRegistry fileRegistry, DeployLogger logger, ModelContext.Properties properties) throws IOException, ParseException { + public static ApplicationBuilder createFromDirectory(String dir, FileRegistry fileRegistry, DeployLogger logger, ModelContext.Properties properties) throws IOException, ParseException { return createFromDirectory(dir, fileRegistry, logger, properties, new RankProfileRegistry()); } - public static SchemaBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { + public static ApplicationBuilder createFromDirectory(String dir, + FileRegistry fileRegistry, + DeployLogger logger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { return createFromDirectory(dir, fileRegistry, logger, properties, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir)); } - private static SchemaBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + private static ApplicationBuilder createFromDirectory(String dir, + FileRegistry fileRegistry, + DeployLogger logger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { return createFromDirectory(dir, MockApplicationPackage.fromSearchDefinitionAndRootDirectory(dir), fileRegistry, logger, properties, rankProfileRegistry, queryProfileRegistry); } - private static SchemaBuilder createFromDirectory(String dir, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - SchemaBuilder builder = new SchemaBuilder(applicationPackage, - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryProfileRegistry); - for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { - builder.importFile(i.next()); + private static ApplicationBuilder createFromDirectory(String dir, + ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { + ApplicationBuilder builder = new ApplicationBuilder(applicationPackage, + fileRegistry, + deployLogger, + properties, + rankProfileRegistry, + queryProfileRegistry); + for (var i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { + builder.addSchemaFile(i.next().toString()); } builder.build(true); return builder; @@ -513,15 +516,15 @@ public class SchemaBuilder { * Convenience factory method to import and build a {@link Schema} object from a raw object. * * @param rawSchema the raw object to build from - * @return the built {@link SchemaBuilder} object - * @see #importRawSchema(Schema) + * @return the built {@link ApplicationBuilder} object + * @see #add(Schema) */ - public static SchemaBuilder createFromRawSchema(Schema rawSchema, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { - SchemaBuilder builder = new SchemaBuilder(rankProfileRegistry, queryProfileRegistry); - builder.importRawSchema(rawSchema); - builder.build(); + public static ApplicationBuilder createFromRawSchema(Schema rawSchema, + RankProfileRegistry rankProfileRegistry, + QueryProfileRegistry queryProfileRegistry) { + ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry, queryProfileRegistry); + builder.add(rawSchema); + builder.build(true); return builder; } @@ -530,7 +533,7 @@ public class SchemaBuilder { * * @param rawSchema the raw object to build from * @return the built {@link Schema} object - * @see #importRawSchema(Schema) + * @see #add(Schema) */ public static Schema buildFromRawSchema(Schema rawSchema, RankProfileRegistry rankProfileRegistry, @@ -550,4 +553,10 @@ public class SchemaBuilder { public DeployLogger getDeployLogger() { return deployLogger; } + @SuppressWarnings("EmptyCatchBlock") + private static void closeIgnoreException(Reader reader) { + try { + reader.close(); + } catch(Exception e) {} + } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java index 56a739ced8b..4444636fbe8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java @@ -40,14 +40,14 @@ public class DefaultRankProfile extends RankProfile { RankSetting setting = super.getRankSetting(fieldOrIndex,type); if (setting != null) return setting; - ImmutableSDField field = getSearch().getConcreteField(fieldOrIndex); + ImmutableSDField field = schema().getConcreteField(fieldOrIndex); if (field != null) { setting = toRankSetting(field,type); if (setting != null) return setting; } - Index index = getSearch().getIndex(fieldOrIndex); + Index index = schema().getIndex(fieldOrIndex); if (index != null) { setting = toRankSetting(index,type); if (setting != null) @@ -89,7 +89,7 @@ public class DefaultRankProfile extends RankProfile { public Set<RankSetting> rankSettings() { Set<RankSetting> settings = new LinkedHashSet<>(20); settings.addAll(this.rankSettings); - for (ImmutableSDField field : getSearch().allConcreteFields() ) { + for (ImmutableSDField field : schema().allConcreteFields() ) { addSetting(field, RankSetting.Type.WEIGHT, settings); addSetting(field, RankSetting.Type.RANKTYPE, settings); addSetting(field, RankSetting.Type.LITERALBOOST, settings); @@ -97,7 +97,7 @@ public class DefaultRankProfile extends RankProfile { } // Foer settings that really pertains to indexes do the explicit indexes too - for (Index index : getSearch().getExplicitIndices()) { + for (Index index : schema().getExplicitIndices()) { addSetting(index, RankSetting.Type.PREFERBITVECTOR, settings); } return settings; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java index 55f24123940..17d9bf635fa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -48,12 +48,12 @@ public class DocumentModelBuilder { private final DocumentModel model; - public DocumentModelBuilder(DocumentModel model) { - this.model = model; - model.getDocumentManager().add(VespaDocumentType.INSTANCE); + public DocumentModelBuilder() { + this.model = new DocumentModel(); + this.model.getDocumentManager().add(VespaDocumentType.INSTANCE); } - public void addToModel(Collection<Schema> schemaList) { + public DocumentModel build(Collection<Schema> schemaList) { List<SDDocumentType> docList = new LinkedList<>(); for (Schema schema : schemaList) { docList.add(schema.getDocument()); @@ -65,6 +65,7 @@ public class DocumentModelBuilder { toAdd = tryAdd(schemaList)) { schemaList = toAdd; } + return model; } private List<SDDocumentType> sortDocumentTypes(List<SDDocumentType> docList) { @@ -124,7 +125,7 @@ public class DocumentModelBuilder { return left; } - public void addToModel(Schema schema) { + private void addToModel(Schema schema) { // Then we add the search specific stuff SearchDef searchDef = new SearchDef(schema.getName()); addSearchFields(schema.extraFieldList(), searchDef); @@ -232,8 +233,7 @@ public class DocumentModelBuilder { @SuppressWarnings("deprecation") private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo, Collection<NewDocumentType> docs, - Set<TypeReplacement> replacements) - { + Set<TypeReplacement> replacements) { DataType original = type; if (type instanceof TemporaryStructuredDataType) { DataType other = repo.getDataType(type.getId()); @@ -298,8 +298,7 @@ public class DocumentModelBuilder { } private static DataType specialHandleAnnotationReferenceRecurse(NewDocumentType docType, String fieldName, - DataType dataType) - { + DataType dataType) { if (dataType instanceof TemporaryAnnotationReferenceDataType) { TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; if (refType.getId() != 0) { @@ -359,6 +358,7 @@ public class DocumentModelBuilder { addType(dt, s); return s; } + private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) { if (sa.getInherits() != null) { AnnotationType tmp = sdoc.findAnnotation(sa.getInherits()); @@ -367,7 +367,7 @@ public class DocumentModelBuilder { } return false; } - @SuppressWarnings("deprecation") + private NewDocumentType convert(SDDocumentType sdoc) { Map<AnnotationType, String> annotationInheritance = new HashMap<>(); Map<StructDataType, String> structInheritance = new HashMap<>(); @@ -463,6 +463,7 @@ public class DocumentModelBuilder { } } } + private static void extractNestedTypes(NewDocumentType dt, DataType type) { if (type instanceof StructDataType) { StructDataType tmp = (StructDataType) type; @@ -484,8 +485,11 @@ public class DocumentModelBuilder { throw new IllegalArgumentException(type.toString()); } } + private static boolean testAddType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, true); } + private static boolean addType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, false); } + private static boolean internalAddType(NewDocumentType dt, DataType type, boolean dryRun) { DataType oldType = dt.getDataTypeRecursive(type.getId()); if (oldType == null) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java index c672b662874..1d71a9f1494 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ModelContext; @@ -14,8 +15,11 @@ import com.yahoo.searchdefinition.document.SDDocumentType; */ public class DocumentOnlySchema extends Schema { - public DocumentOnlySchema(Application application, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - super(application, fileRegistry, deployLogger, properties); + public DocumentOnlySchema(ApplicationPackage applicationPackage, + FileRegistry fileRegistry, + DeployLogger deployLogger, + ModelContext.Properties properties) { + super(applicationPackage, fileRegistry, deployLogger, properties); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 5f6bde3b791..1b6c4aa9459 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -57,11 +57,12 @@ public class RankProfile implements Cloneable { public final static String FIRST_PHASE = "firstphase"; public final static String SECOND_PHASE = "secondphase"; + /** The search definition-unique name of this rank profile */ private final String name; - /** The search definition owning this profile, or null if global (owned by a model) */ - private final ImmutableSchema search; + /** The schema owning this profile, or null if global (owned by a model) */ + private final ImmutableSchema schema; /** The name of the rank profile inherited by this */ private String inheritedName = null; @@ -132,20 +133,6 @@ public class RankProfile implements Cloneable { private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; - private static class CachedFunctions { - private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; - private final ImmutableMap<String, ExpressionFunction> allExpressionFunctions; - CachedFunctions(Map<String, RankingExpressionFunction> functions) { - allRankingExpressionFunctions = functions; - ImmutableMap.Builder<String,ExpressionFunction> mapBuilder = new ImmutableMap.Builder<>(); - for (var entry : functions.entrySet()) { - ExpressionFunction function = entry.getValue().function(); - mapBuilder.put(function.getName(), function); - } - allExpressionFunctions = mapBuilder.build(); - } - } - /** * Creates a new rank profile for a particular search definition * @@ -156,7 +143,7 @@ public class RankProfile implements Cloneable { */ public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { this.name = Objects.requireNonNull(name, "name cannot be null"); - this.search = Objects.requireNonNull(schema, "search cannot be null"); + this.schema = Objects.requireNonNull(schema, "search cannot be null"); this.onnxModels = null; this.rankingConstants = rankingConstants; this.rankProfileRegistry = rankProfileRegistry; @@ -172,7 +159,7 @@ public class RankProfile implements Cloneable { public RankProfile(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants, OnnxModels onnxModels) { this.name = Objects.requireNonNull(name, "name cannot be null"); - this.search = null; + this.schema = null; this.rankProfileRegistry = rankProfileRegistry; this.rankingConstants = rankingConstants; this.onnxModels = onnxModels; @@ -180,10 +167,10 @@ public class RankProfile implements Cloneable { this.deployLogger = deployLogger; } - public String getName() { return name; } + public String name() { return name; } /** Returns the search definition owning this, or null if it is global */ - public ImmutableSchema getSearch() { return search; } + public ImmutableSchema schema() { return schema; } /** Returns the application this is part of */ public ApplicationPackage applicationPackage() { @@ -196,19 +183,19 @@ public class RankProfile implements Cloneable { } public Map<String, OnnxModel> onnxModels() { - return search != null ? search.onnxModels().asMap() : onnxModels.asMap(); + return schema != null ? schema.onnxModels().asMap() : onnxModels.asMap(); } private Stream<ImmutableSDField> allFields() { - if (search == null) return Stream.empty(); + if (schema == null) return Stream.empty(); if (allFieldsList == null) { - allFieldsList = search.allFieldsList(); + allFieldsList = schema.allFieldsList(); } return allFieldsList.stream(); } private Stream<ImmutableSDField> allImportedFields() { - return search != null ? search.allImportedFields() : Stream.empty(); + return schema != null ? schema.allImportedFields() : Stream.empty(); } /** @@ -228,9 +215,9 @@ public class RankProfile implements Cloneable { if (inherited == null) { inherited = resolveInherited(); if (inherited == null) { - String msg = "rank-profile '" + getName() + "' inherits '" + inheritedName + - "', but it does not exist anywhere in the inheritance of search '" + - ((getSearch() != null) ? getSearch().getName() : " global rank profiles") + "'."; + String msg = "rank-profile '" + name() + "' inherits '" + inheritedName + + "', but this is not found in " + + ((schema() != null) ? schema() : " global rank profiles"); throw new IllegalArgumentException(msg); } else { List<String> children = new ArrayList<>(); @@ -242,9 +229,9 @@ public class RankProfile implements Cloneable { } private String createFullyQualifiedName() { - return (search != null) - ? (search.getName() + "." + getName()) - : getName(); + return (schema != null) + ? (schema.getName() + "." + name()) + : name(); } private void verifyNoInheritanceCycle(List<String> children, RankProfile parent) { @@ -275,8 +262,8 @@ public class RankProfile implements Cloneable { private RankProfile resolveInherited() { if (inheritedName == null) return null; - return (getSearch() != null) - ? resolveInherited(search) + return (schema() != null) + ? resolveInherited(schema) : rankProfileRegistry.getGlobal(inheritedName); } @@ -289,7 +276,7 @@ public class RankProfile implements Cloneable { public boolean inherits(String name) { RankProfile parent = getInherited(); while (parent != null) { - if (parent.getName().equals(name)) + if (parent.name().equals(name)) return true; parent = parent.getInherited(); } @@ -619,7 +606,7 @@ public class RankProfile implements Cloneable { @Override public String toString() { - return "rank profile '" + getName() + "'"; + return "rank profile '" + name() + "'"; } public int getRerankCount() { @@ -819,14 +806,14 @@ public class RankProfile implements Cloneable { private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { if (expression.trim().length() == 0) - throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + name + "."); + throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { return new ExpressionFunction(name, arguments, new RankingExpression(name, rankingExpressionReader)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { ParseException exception = new ParseException("Could not parse ranking expression '" + expression.trim() + - "' in " + getName()+ ", " + name + "."); + "' in " + name() + ", " + name + "."); throw (ParseException)exception.initCause(e); } catch (IOException e) { @@ -848,10 +835,10 @@ public class RankProfile implements Cloneable { String fileName = extractFileName(expression); File file = new File(fileName); if (!file.isAbsolute() && file.getPath().contains("/")) // See ticket 4102122 - throw new IllegalArgumentException("In " + getName() + ", " + expName + ", ranking references file '" + file + - "' in subdirectory, which is not supported."); + throw new IllegalArgumentException("In " + name() + ", " + expName + ", ranking references file '" + file + + "' in subdirectory, which is not supported."); - return search.getRankingExpression(fileName); + return schema.getRankingExpression(fileName); } /** Shallow clones this */ @@ -888,7 +875,7 @@ public class RankProfile implements Cloneable { return compiled; } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Rank profile '" + getName() + "' is invalid", e); + throw new IllegalArgumentException("Rank profile '" + name() + "' is invalid", e); } } @@ -1321,4 +1308,22 @@ public class RankProfile implements Cloneable { } + private static class CachedFunctions { + + private final Map<String, RankingExpressionFunction> allRankingExpressionFunctions; + + private final ImmutableMap<String, ExpressionFunction> allExpressionFunctions; + + CachedFunctions(Map<String, RankingExpressionFunction> functions) { + allRankingExpressionFunctions = functions; + ImmutableMap.Builder<String,ExpressionFunction> mapBuilder = new ImmutableMap.Builder<>(); + for (var entry : functions.entrySet()) { + ExpressionFunction function = entry.getValue().function(); + mapBuilder.put(function.getName(), function); + } + allExpressionFunctions = mapBuilder.build(); + } + + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java index 08ae3d838ec..75c3aa313c0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java @@ -6,7 +6,6 @@ import com.yahoo.searchdefinition.document.SDDocumentType; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -44,22 +43,22 @@ public class RankProfileRegistry { /** Adds a rank profile to this registry */ public void add(RankProfile rankProfile) { - String searchName = extractName(rankProfile.getSearch()); - if ( ! rankProfiles.containsKey(searchName)) { - rankProfiles.put(searchName, new LinkedHashMap<>()); + String schemaName = extractName(rankProfile.schema()); + if ( ! rankProfiles.containsKey(schemaName)) { + rankProfiles.put(schemaName, new LinkedHashMap<>()); } checkForDuplicate(rankProfile); - rankProfiles.get(searchName).put(rankProfile.getName(), rankProfile); + rankProfiles.get(schemaName).put(rankProfile.name(), rankProfile); } private void checkForDuplicate(RankProfile rankProfile) { - String rankProfileName = rankProfile.getName(); - RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.getSearch())).get(rankProfileName); + String rankProfileName = rankProfile.name(); + RankProfile existingRankProfileWithSameName = rankProfiles.get(extractName(rankProfile.schema())).get(rankProfileName); if (existingRankProfileWithSameName == null) return; if ( ! overridableRankProfileNames.contains(rankProfileName)) { throw new IllegalArgumentException("Duplicate rank profile '" + rankProfileName + "' in " + - rankProfile.getSearch()); + rankProfile.schema()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java index ddad67324ba..4df118545eb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java @@ -55,6 +55,9 @@ public class Schema implements ImmutableSchema { /** The unique name of this schema */ private String name; + /** The application package this is constructed from */ + private final ApplicationPackage applicationPackage; + /** The name of the schema this should inherit all the content of, if any */ private final Optional<String> inherited; @@ -93,21 +96,22 @@ public class Schema implements ImmutableSchema { /** The resulting processed field */ private Optional<ImportedFields> importedFields = Optional.empty(); - private final Application owner; private final DeployLogger deployLogger; private final ModelContext.Properties properties; + private Application owner; + /** Testing only */ - public Schema(String name) { - this(name, Optional.empty(), null, null, new BaseDeployLogger(), new TestProperties()); + public Schema(String name, ApplicationPackage applicationPackage) { + this(name, applicationPackage, Optional.empty(), null, new BaseDeployLogger(), new TestProperties()); } public Schema(String name, - Application application, + ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - this(name, Optional.empty(), application, fileRegistry, deployLogger, properties); + this(name, applicationPackage, Optional.empty(), fileRegistry, deployLogger, properties); } /** @@ -115,30 +119,30 @@ public class Schema implements ImmutableSchema { * * @param name of the schema * @param inherited the schema this inherits, if any - * @param application the application containing this */ public Schema(String name, + ApplicationPackage applicationPackage, Optional<String> inherited, - Application application, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - this(inherited, application, fileRegistry, deployLogger, properties, false); - this.name = name; + this(inherited, applicationPackage, fileRegistry, deployLogger, properties, false); + this.name = Objects.requireNonNull(name, "A schema must have a name"); } - protected Schema(Application application, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties) { - this(Optional.empty(), application, fileRegistry, deployLogger, properties, true); + protected Schema(ApplicationPackage applicationPackage, FileRegistry fileRegistry, + DeployLogger deployLogger, ModelContext.Properties properties) { + this(Optional.empty(), applicationPackage, fileRegistry, deployLogger, properties, true); } private Schema(Optional<String> inherited, - Application application, + ApplicationPackage applicationPackage, FileRegistry fileRegistry, DeployLogger deployLogger, ModelContext.Properties properties, boolean documentsOnly) { this.inherited = inherited; - this.owner = application; + this.applicationPackage = applicationPackage; this.deployLogger = deployLogger; this.properties = properties; this.documentsOnly = documentsOnly; @@ -147,14 +151,21 @@ public class Schema implements ImmutableSchema { onnxModels = new OnnxModels(fileRegistry, Optional.of(this)); } - protected void setName(String name) { - this.name = name; + /** + * Assigns the owner of this + * + * @throws IllegalStateException if an owner is already assigned + */ + public void setOwner(Application owner) { + if (this.owner != null) + throw new IllegalStateException("Cannot reassign the owner of " + this); + this.owner = owner; } + protected void setName(String name) { this.name = name; } + @Override - public String getName() { - return name; - } + public String getName() {return name; } /** Returns true if this only defines a document type, not a full schema */ public boolean isDocumentsOnly() { @@ -311,16 +322,13 @@ public class Schema implements ImmutableSchema { */ @Override public Reader getRankingExpression(String fileName) { - return owner.applicationPackage().getRankingExpression(fileName); + return applicationPackage.getRankingExpression(fileName); } public Application application() { return owner; } @Override - public ApplicationPackage applicationPackage() { - if (owner == null) return null; - return owner.applicationPackage(); - } + public ApplicationPackage applicationPackage() { return applicationPackage; } @Override public DeployLogger getDeployLogger() { return deployLogger; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java index 14e303522e0..bcdc9e656e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.SchemaBuilder; +import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; @@ -11,22 +11,21 @@ import java.util.Collections; import java.util.List; /** - * Auxiliary facade for deriving configs from search definitions + * Facade for deriving configs from schemas * * @author bratseth */ public class Deriver { - public static SchemaBuilder getSearchBuilder(List<String> sds) { - SchemaBuilder builder = new SchemaBuilder(); + public static ApplicationBuilder getSchemaBuilder(List<String> schemas) { + ApplicationBuilder builder = new ApplicationBuilder(); try { - for (String s : sds) { - builder.importFile(s); - } + for (String schema : schemas) + builder.addSchemaFile(schema); } catch (ParseException | IOException e) { throw new IllegalArgumentException(e); } - builder.build(); + builder.build(true); return builder; } @@ -34,22 +33,22 @@ public class Deriver { return getDocumentManagerConfig(Collections.singletonList(sd)); } - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> sds) { - return new DocumentManager().produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder()); + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> schemas) { + return new DocumentManager().produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); } - public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> sds, boolean useV8DocManagerCfg) { + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> schemas, boolean useV8DocManagerCfg) { return new DocumentManager() .useV8DocManagerCfg(useV8DocManagerCfg) - .produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder()); + .produce(getSchemaBuilder(schemas).getModel(), new DocumentmanagerConfig.Builder()); } - public static DocumenttypesConfig.Builder getDocumentTypesConfig(String sd) { - return getDocumentTypesConfig(Collections.singletonList(sd)); + public static DocumenttypesConfig.Builder getDocumentTypesConfig(String schema) { + return getDocumentTypesConfig(Collections.singletonList(schema)); } - public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> sds) { - return new DocumentTypes().produce(getSearchBuilder(sds).getModel(), new DocumenttypesConfig.Builder()); + public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> schemas) { + return new DocumentTypes().produce(getSchemaBuilder(schemas).getModel(), new DocumenttypesConfig.Builder()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 3081637c975..86c1d478974 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -75,7 +75,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry) { return (rank.getInheritedName() == null) || rankProfiles.containsKey(rank.getInheritedName()) || - (rank.getSearch() != null && registry.resolve(rank.getSearch().getDocument(), rank.getInheritedName()) != null); + (rank.schema() != null && registry.resolve(rank.schema().getDocument(), rank.getInheritedName()) != null); } private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, @@ -92,7 +92,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } Map<String, RankProfile> remaining = new LinkedHashMap<>(); - rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.getName(), rank)); + rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); remaining.remove("default"); while (!remaining.isEmpty()) { List<RankProfile> ready = new ArrayList<>(); @@ -100,7 +100,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ if (areDependenciesReady(rank, rankProfileRegistry)) ready.add(rank); }); processRankProfiles(ready, queryProfiles, importedModels, schema, attributeFields, deployProperties, executor); - ready.forEach(rank -> remaining.remove(rank.getName())); + ready.forEach(rank -> remaining.remove(rank.name())); } } private void processRankProfiles(List<RankProfile> ready, @@ -116,8 +116,8 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ onnxModels.add(rank.onnxModels()); } - futureRawRankProfiles.put(rank.getName(), executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties))); + futureRawRankProfiles.put(rank.name(), executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties))); } try { for (Future<RawRankProfile> rawFuture : futureRawRankProfiles.values()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index f775c4e697d..a64f0939677 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -62,7 +62,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { public RawRankProfile(RankProfile rankProfile, LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - this.name = rankProfile.getName(); + this.name = rankProfile.name(); compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), attributeFields, deployProperties).derive(largeExpressions)); } @@ -158,7 +158,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { */ Deriver(RankProfile compiled, AttributeFields attributeFields, ModelContext.Properties deployProperties) { - rankprofileName = compiled.getName(); + rankprofileName = compiled.name(); attributeTypes = compiled.getAttributeTypes(); queryFeatureTypes = compiled.getQueryFeatureTypes(); firstPhaseRanking = compiled.getFirstPhaseRanking(); @@ -447,8 +447,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { } private void deriveOnnxModelFunctionsAndFeatures(RankProfile rankProfile) { - if (rankProfile.getSearch() == null) return; - if (rankProfile.getSearch().onnxModels().asMap().isEmpty()) return; + if (rankProfile.schema() == null) return; + if (rankProfile.schema().onnxModels().asMap().isEmpty()) return; replaceOnnxFunctionInputs(rankProfile); replaceImplicitOnnxConfigFeatures(summaryFeatures, rankProfile); replaceImplicitOnnxConfigFeatures(matchFeatures, rankProfile); @@ -457,7 +457,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void replaceOnnxFunctionInputs(RankProfile rankProfile) { Set<String> functionNames = rankProfile.getFunctions().keySet(); if (functionNames.isEmpty()) return; - for (OnnxModel onnxModel: rankProfile.getSearch().onnxModels().asMap().values()) { + for (OnnxModel onnxModel: rankProfile.schema().onnxModels().asMap().values()) { for (Map.Entry<String, String> mapping : onnxModel.getInputMap().entrySet()) { String source = mapping.getValue(); if (functionNames.contains(source)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java index 96f2f2f1d24..b4d76445507 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java @@ -29,11 +29,7 @@ public class TemporaryImportedFields { } public boolean hasField(String fieldName) { - boolean has = fields.get(fieldName) != null; - if (has) return true; - if (owner.inherited().isEmpty()) return false; - if (owner.inherited().get().temporaryImportedFields().isEmpty()) return false; - return owner.inherited().get().temporaryImportedFields().get().hasField(fieldName); + return fields.get(fieldName) != null; } public Map<String, TemporaryImportedField> fields() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java index 6d9f4cdec92..35ee9ddb9ed 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java @@ -49,12 +49,12 @@ public class OnnxModelTransformer extends ExpressionTransformer<RankProfileTrans private ExpressionNode transformFeature(ReferenceNode feature, RankProfileTransformContext context) { if (context.rankProfile() == null) return feature; - if (context.rankProfile().getSearch() == null) return feature; + if (context.rankProfile().schema() == null) return feature; return transformFeature(feature, context.rankProfile()); } public static ExpressionNode transformFeature(ReferenceNode feature, RankProfile rankProfile) { - ImmutableSchema search = rankProfile.getSearch(); + ImmutableSchema search = rankProfile.schema(); final String featureName = feature.getName(); if ( ! featureName.equals("onnxModel") && ! featureName.equals("onnx")) return feature; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java index cde172d00b9..0b275c6a722 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java @@ -6,11 +6,11 @@ import com.yahoo.javacc.FastCharStream; /** * @author Simon Thoresen Hult */ -@SuppressWarnings("deprecation") public class SimpleCharStream extends FastCharStream implements com.yahoo.searchdefinition.parser.CharStream, - com.yahoo.vespa.indexinglanguage.parser.CharStream -{ + com.yahoo.vespa.indexinglanguage.parser.CharStream { + public SimpleCharStream(String input) { super(input); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java index 5643bb660f1..3759fc453df 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java @@ -30,7 +30,7 @@ public class DiversitySettingsValidator extends Processor { } private void validate(RankProfile rankProfile, RankProfile.DiversitySettings settings) { String attributeName = settings.getAttribute(); - new AttributeValidator(schema.getName(), rankProfile.getName(), + new AttributeValidator(schema.getName(), rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java index d89f83c333f..3f97bf83565 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java @@ -63,7 +63,7 @@ public class FilterFieldNames extends Processor { filterFields.add(fieldName); } } else { - deployLogger.logApplicationPackage(Level.WARNING, "For rank profile '" + profile.getName() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'"); + deployLogger.logApplicationPackage(Level.WARNING, "For rank profile '" + profile.name() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'"); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java index b697c584ece..4eae6b47833 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java @@ -35,7 +35,7 @@ public class MatchPhaseSettingsValidator extends Processor { private void validateMatchPhaseSettings(RankProfile rankProfile, RankProfile.MatchPhaseSettings settings) { String attributeName = settings.getAttribute(); new AttributeValidator(schema.getName(), - rankProfile.getName(), + rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java index f5c3fa35e34..e6adb8b683b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java @@ -89,10 +89,10 @@ public class RankingExpressionTypeResolver extends Processor { if ( context.tensorsAreUsed() && ! context.queryFeaturesNotDeclared().isEmpty() && ! warnedAbout.containsAll(context.queryFeaturesNotDeclared())) { - deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in '" + profile.getName() + - "' are not declared in query profile " + - "types and will be interpreted as scalars, not tensors: " + - context.queryFeaturesNotDeclared()); + deployLogger.logApplicationPackage(Level.WARNING, "The following query features used in '" + profile.name() + + "' are not declared in query profile " + + "types and will be interpreted as scalars, not tensors: " + + context.queryFeaturesNotDeclared()); warnedAbout.addAll(context.queryFeaturesNotDeclared()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java index 8194b9f9e06..f4f920d9ec8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java @@ -35,9 +35,9 @@ public class ReservedFunctionNames extends Processor { for (String functionName : rp.getFunctions().keySet()) { if (reservedNames.contains(functionName)) { deployLogger.logApplicationPackage(Level.WARNING, "Function '" + functionName + "' " + - "in rank profile '" + rp.getName() + "' " + - "has a reserved name. This might mean that the function shadows " + - "the built-in function with the same name." + "in rank profile '" + rp.name() + "' " + + "has a reserved name. This might mean that the function shadows " + + "the built-in function with the same name." ); } } |