From 7cfec9da18f7e1fc5d23cafef64c5f7b7af3750f Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Mon, 25 Oct 2021 12:34:56 +0200 Subject: SearchBuilder -> SchemaBuilder --- .../com/yahoo/searchdefinition/SearchBuilder.java | 555 --------------------- 1 file changed, 555 deletions(-) delete mode 100644 config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java (limited to 'config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java') diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java deleted file mode 100644 index dc5ab752f31..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java +++ /dev/null @@ -1,555 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.search.query.profile.config.QueryProfileXMLReader; -import com.yahoo.searchdefinition.derived.SearchOrderer; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.parser.SDParser; -import com.yahoo.searchdefinition.parser.SimpleCharStream; -import com.yahoo.searchdefinition.parser.TokenMgrException; -import com.yahoo.searchdefinition.processing.Processing; -import com.yahoo.searchdefinition.processing.Processor; -import com.yahoo.vespa.documentmodel.DocumentModel; -import com.yahoo.vespa.model.container.search.QueryProfiles; -import com.yahoo.yolean.Exceptions; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -/** - * Helper class for importing {@link Schema} objects in an unambiguous way. The pattern for using this is to 1) Import - * all available search definitions, using the importXXX() methods, 2) provide the available rank types and rank - * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built - * search objects using the {@link #getSearch(String)} method. - */ -// Since this was created we have added Application, and much of the content in this should probably migrate there. -public class SearchBuilder { - - private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); - private final DocumentModel model = new DocumentModel(); - private final Application application; - private final QueryProfileRegistry queryProfileRegistry; - private final FileRegistry fileRegistry; - private final DeployLogger deployLogger; - private final ModelContext.Properties properties; - /** True to build the document aspect only, skipping instantiation of rank profiles */ - private final boolean documentsOnly; - - private List schemaList = new LinkedList<>(); - private boolean isBuilt = false; - - private final Set> processorsToSkip = new HashSet<>(); - - /** For testing only */ - public SearchBuilder() { - this(new RankProfileRegistry(), new QueryProfileRegistry()); - } - - /** For testing only */ - public SearchBuilder(DeployLogger deployLogger) { - this(MockApplicationPackage.createEmpty(), deployLogger); - } - - /** For testing only */ - public SearchBuilder(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { - this(MockApplicationPackage.createEmpty(), deployLogger, rankProfileRegistry); - } - - /** Used for generating documents for typed access to document fields in Java */ - public SearchBuilder(boolean documentsOnly) { - this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry(), documentsOnly); - } - - /** For testing only */ - public SearchBuilder(ApplicationPackage app, DeployLogger deployLogger) { - this(app, new MockFileRegistry(), deployLogger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); - } - - /** For testing only */ - public SearchBuilder(ApplicationPackage app, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { - this(app, new MockFileRegistry(), deployLogger, new TestProperties(), rankProfileRegistry, new QueryProfileRegistry()); - } - - /** For testing only */ - public SearchBuilder(RankProfileRegistry rankProfileRegistry) { - this(rankProfileRegistry, new QueryProfileRegistry()); - } - - /** For testing only */ - public SearchBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry) { - this(rankProfileRegistry, queryProfileRegistry, new TestProperties()); - } - public SearchBuilder(RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfileRegistry, ModelContext.Properties properties) { - this(MockApplicationPackage.createEmpty(), new MockFileRegistry(), new BaseDeployLogger(), properties, rankProfileRegistry, queryProfileRegistry); - } - - public SearchBuilder(ApplicationPackage app, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { - this(app, fileRegistry, deployLogger, properties, rankProfileRegistry, queryProfileRegistry, false); - } - private SearchBuilder(ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry, - boolean documentsOnly) { - this.application = new Application(applicationPackage, rankProfileRegistry); - this.queryProfileRegistry = queryProfileRegistry; - this.fileRegistry = fileRegistry; - this.deployLogger = deployLogger; - this.properties = properties; - this.documentsOnly = documentsOnly; - } - - /** - * Import search definition. - * - * @param fileName the name of the file to import - * @return the name of the imported object - * @throws IOException thrown if the file can not be read for some reason - * @throws ParseException thrown if the file does not contain a valid search definition - */ - public String importFile(String fileName) throws IOException, ParseException { - File file = new File(fileName); - return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent()); - } - - private String importFile(Path file) throws IOException, ParseException { - return importFile(file.toString()); - } - - /** - * Reads and parses the search definition string provided by the given reader. Once all search definitions have been - * imported, call {@link #build()}. - * - * @param reader the reader whose content to import - * @param searchDefDir the path to use when resolving file references - * @return the name of the imported object - * @throws ParseException thrown if the file does not contain a valid search definition - */ - public String importReader(NamedReader reader, String searchDefDir) throws IOException, ParseException { - return importString(IOUtils.readAll(reader), searchDefDir); - } - - /** - * Import search definition. - * - * @param str the string to parse. - * @return the name of the imported object. - * @throws ParseException thrown if the file does not contain a valid search definition. - */ - public String importString(String str) throws ParseException { - return importString(str, null); - } - - private String importString(String str, String searchDefDir) throws ParseException { - SimpleCharStream stream = new SimpleCharStream(str); - try { - return importRawSearch(new SDParser(stream, fileRegistry, deployLogger, properties, application, - application.rankProfileRegistry(), documentsOnly) - .schema(docTypeMgr, searchDefDir)); - } catch (TokenMgrException e) { - throw new ParseException("Unknown symbol: " + e.getMessage()); - } catch (ParseException pe) { - throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); - } - } - - /** - * Registers the given search object to the internal list of objects to be processed during {@link #build()}. A - * {@link Schema} object is considered to be "raw" if it has not already been processed. This is the case for most - * programmatically constructed search objects used in unit tests. - * - * @param schema the object to import. - * @return the name of the imported object. - * @throws IllegalArgumentException if the given search object has already been processed. - */ - public String importRawSearch(Schema schema) { - if (schema.getName() == null) - throw new IllegalArgumentException("Schema has no name"); - String rawName = schema.getName(); - application.add(schema); - schemaList.add(schema); - return rawName; - } - - /** - * Only for testing. - * - * Processes and finalizes the imported search definitions so that they become available through the {@link - * #getSearch(String)} method. - * - * @throws IllegalStateException Thrown if this method has already been called. - */ - public void build() { - build(true); - } - - /** - * Processes and finalizes the imported search definitions so that they become available through the {@link - * #getSearch(String)} method. - * - * @throws IllegalStateException Thrown if this method has already been called. - */ - public void build(boolean validate) { - if (isBuilt) throw new IllegalStateException("Model already built"); - - if (validate) - application.validate(deployLogger); - - List built = new ArrayList<>(); - List sdocs = new ArrayList<>(); - sdocs.add(SDDocumentType.VESPA_DOCUMENT); - for (Schema schema : schemaList) { - if (schema.hasDocument()) { - sdocs.add(schema.getDocument()); - } - } - - var orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); - orderer.process(); - for (SDDocumentType sdoc : orderer.getOrdered()) { - new FieldOperationApplierForStructs().process(sdoc); - new FieldOperationApplier().process(sdoc); - } - - var resolver = new DocumentReferenceResolver(schemaList); - sdocs.forEach(resolver::resolveReferences); - sdocs.forEach(resolver::resolveInheritedReferences); - var importedFieldsEnumerator = new ImportedFieldsEnumerator(schemaList); - sdocs.forEach(importedFieldsEnumerator::enumerateImportedFields); - - if (validate) - new DocumentGraphValidator().validateDocumentGraph(sdocs); - - var builder = new DocumentModelBuilder(model); - for (Schema schema : new SearchOrderer().order(schemaList)) { - new FieldOperationApplierForSearch().process(schema); // TODO: Why is this not in the regular list? - process(schema, new QueryProfiles(queryProfileRegistry, deployLogger), validate); - built.add(schema); - } - builder.addToModel(schemaList); - - if ( validate && ! builder.valid() ) - throw new IllegalArgumentException("Impossible to build a correct model"); - - schemaList = built; - isBuilt = true; - } - - /** Returns a modifiable set of processors we should skip for these schemas. Useful for testing. */ - public Set> processorsToSkip() { return processorsToSkip; } - - /** - * Processes and returns the given {@link Schema} object. This method has been factored out of the {@link - * #build()} method so that subclasses can choose not to build anything. - */ - private void process(Schema schema, QueryProfiles queryProfiles, boolean validate) { - new Processing().process(schema, deployLogger, application.rankProfileRegistry(), queryProfiles, validate, - documentsOnly, processorsToSkip); - } - - /** - * Convenience method to call {@link #getSearch(String)} when there is only a single {@link Schema} object - * built. This method will never return null. - * - * @return the built object - * @throws IllegalStateException if there is not exactly one search. - */ - public Schema getSearch() { - if ( ! isBuilt) throw new IllegalStateException("Searches not built."); - if (application.schemas().size() != 1) - throw new IllegalStateException("This call only works if we have 1 schema. Schemas: " + - application.schemas().values()); - - return application.schemas().values().stream().findAny().get(); - } - - public DocumentModel getModel() { - return model; - } - - /** - * Returns the built {@link Schema} object that has the given name. If the name is unknown, this method will simply - * return null. - * - * @param name the name of the search definition to return, - * or null to return the only one or throw an exception if there are multiple to choose from - * @return the built object, or null if none with this name - * @throws IllegalStateException if {@link #build()} has not been called. - */ - public Schema getSearch(String name) { - if ( ! isBuilt) throw new IllegalStateException("Searches not built."); - if (name == null) return getSearch(); - return application.schemas().get(name); - } - - public Application application() { return application; } - - /** - * Convenience method to return a list of all built {@link Schema} objects. - * - * @return the list of built searches - */ - public List getSearchList() { - return new ArrayList<>(schemaList); - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a string. - * - * @param sd The string to build from. - * @return The built {@link SearchBuilder} object. - * @throws ParseException Thrown if there was a problem parsing the string. - */ - public static SearchBuilder createFromString(String sd) throws ParseException { - return createFromString(sd, new BaseDeployLogger()); - } - - public static SearchBuilder createFromString(String sd, DeployLogger logger) throws ParseException { - SearchBuilder builder = new SearchBuilder(logger); - builder.importString(sd); - builder.build(true); - return builder; - } - - public static SearchBuilder createFromStrings(DeployLogger logger, String ... schemas) throws ParseException { - SearchBuilder builder = new SearchBuilder(logger); - for (var schema : schemas) - builder.importString(schema); - builder.build(true); - return builder; - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a file. Only for testing. - * - * @param fileName the file to build from - * @return the built {@link SearchBuilder} object - * @throws IOException if there was a problem reading the file. - * @throws ParseException if there was a problem parsing the file content. - */ - public static SearchBuilder createFromFile(String fileName) throws IOException, ParseException { - return createFromFile(fileName, new BaseDeployLogger()); - } - - /** - * Convenience factory methdd to create a SearchBuilder from multiple SD files. Only for testing. - */ - public static SearchBuilder createFromFiles(Collection fileNames) throws IOException, ParseException { - return createFromFiles(fileNames, new BaseDeployLogger()); - } - - public static SearchBuilder createFromFile(String fileName, DeployLogger logger) throws IOException, ParseException { - return createFromFile(fileName, logger, new RankProfileRegistry(), new QueryProfileRegistry()); - } - - private static SearchBuilder createFromFiles(Collection fileNames, DeployLogger logger) throws IOException, ParseException { - return createFromFiles(fileNames, new MockFileRegistry(), logger, new TestProperties(), new RankProfileRegistry(), new QueryProfileRegistry()); - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a file. - * - * @param fileName the file to build from. - * @param deployLogger logger for deploy messages. - * @param rankProfileRegistry registry for rank profiles. - * @return the built {@link SearchBuilder} object. - * @throws IOException if there was a problem reading the file. - * @throws ParseException if there was a problem parsing the file content. - */ - private static SearchBuilder createFromFile(String fileName, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) - throws IOException, ParseException { - return createFromFiles(Collections.singletonList(fileName), new MockFileRegistry(), deployLogger, new TestProperties(), - rankProfileRegistry, queryprofileRegistry); - } - - /** - * Convenience factory methdd to create a SearchBuilder from multiple SD files.. - */ - private static SearchBuilder createFromFiles(Collection fileNames, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryprofileRegistry) - throws IOException, ParseException { - SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(), - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryprofileRegistry); - for (String fileName : fileNames) { - builder.importFile(fileName); - } - builder.build(true); - return builder; - } - - - public static SearchBuilder createFromDirectory(String dir, FileRegistry fileRegistry, DeployLogger logger, ModelContext.Properties properties) throws IOException, ParseException { - return createFromDirectory(dir, fileRegistry, logger, properties, new RankProfileRegistry()); - } - public static SearchBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { - return createFromDirectory(dir, fileRegistry, logger, properties, rankProfileRegistry, createQueryProfileRegistryFromDirectory(dir)); - } - private static SearchBuilder createFromDirectory(String dir, - FileRegistry fileRegistry, - DeployLogger logger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - return createFromDirectory(dir, MockApplicationPackage.fromSearchDefinitionAndRootDirectory(dir), fileRegistry, logger, properties, - rankProfileRegistry, queryProfileRegistry); - } - - private static SearchBuilder createFromDirectory(String dir, - ApplicationPackage applicationPackage, - FileRegistry fileRegistry, - DeployLogger deployLogger, - ModelContext.Properties properties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) throws IOException, ParseException { - SearchBuilder builder = new SearchBuilder(applicationPackage, - fileRegistry, - deployLogger, - properties, - rankProfileRegistry, - queryProfileRegistry); - for (Iterator i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { - builder.importFile(i.next()); - } - builder.build(true); - return builder; - } - - private static QueryProfileRegistry createQueryProfileRegistryFromDirectory(String dir) { - File queryProfilesDir = new File(dir, "query-profiles"); - if ( ! queryProfilesDir.exists()) return new QueryProfileRegistry(); - return new QueryProfileXMLReader().read(queryProfilesDir.toString()); - } - - // TODO: The build methods below just call the create methods above - remove - - /** - * Convenience factory method to import and build a {@link Schema} object from a file. Only for testing. - * - * @param fileName The file to build from. - * @return The built {@link Schema} object. - * @throws IOException Thrown if there was a problem reading the file. - * @throws ParseException Thrown if there was a problem parsing the file content. - */ - public static Schema buildFromFile(String fileName) throws IOException, ParseException { - return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry(), new QueryProfileRegistry()); - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a file. - * - * @param fileName The file to build from. - * @param rankProfileRegistry Registry for rank profiles. - * @return The built {@link Schema} object. - * @throws IOException Thrown if there was a problem reading the file. - * @throws ParseException Thrown if there was a problem parsing the file content. - */ - public static Schema buildFromFile(String fileName, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) - throws IOException, ParseException { - return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry, queryProfileRegistry); - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a file. - * - * @param fileName The file to build from. - * @param deployLogger Logger for deploy messages. - * @param rankProfileRegistry Registry for rank profiles. - * @return The built {@link Schema} object. - * @throws IOException Thrown if there was a problem reading the file. - * @throws ParseException Thrown if there was a problem parsing the file content. - */ - public static Schema buildFromFile(String fileName, - DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) - throws IOException, ParseException { - return createFromFile(fileName, deployLogger, rankProfileRegistry, queryProfileRegistry).getSearch(); - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a raw object. - * - * @param rawSchema the raw object to build from. - * @return the built {@link SearchBuilder} object. - * @see #importRawSearch(Schema) - */ - public static SearchBuilder createFromRawSearch(Schema rawSchema, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { - SearchBuilder builder = new SearchBuilder(rankProfileRegistry, queryProfileRegistry); - builder.importRawSearch(rawSchema); - builder.build(); - return builder; - } - - /** - * Convenience factory method to import and build a {@link Schema} object from a raw object. - * - * @param rawSchema The raw object to build from. - * @return The built {@link Schema} object. - * @see #importRawSearch(Schema) - */ - public static Schema buildFromRawSearch(Schema rawSchema, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfileRegistry) { - return createFromRawSearch(rawSchema, rankProfileRegistry, queryProfileRegistry).getSearch(); - } - - public RankProfileRegistry getRankProfileRegistry() { - return application.rankProfileRegistry(); - } - - public QueryProfileRegistry getQueryProfileRegistry() { - return queryProfileRegistry; - } - public ModelContext.Properties getProperties() { return properties; } - public DeployLogger getDeployLogger() { return deployLogger; } - -} -- cgit v1.2.3