diff options
author | Jon Bratseth <bratseth@gmail.com> | 2022-05-11 13:12:15 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@gmail.com> | 2022-05-11 13:12:15 +0200 |
commit | a754c48160b4f54f7788679b5f87237f214e6b98 (patch) | |
tree | 9d8cab2f0fa4270fc93f4ca776b86fe2535bfce0 /config-model/src | |
parent | 57d93307b70651fa6fd74f1fabbf15dadc53841e (diff) |
Test ranking constant config generation
Diffstat (limited to 'config-model/src')
17 files changed, 184 insertions, 80 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 1838f1e36b7..6351232b789 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -6,9 +6,6 @@ import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.component.Version; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -18,9 +15,6 @@ import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.config.application.api.ApplicationPackage; @@ -37,7 +31,9 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -53,13 +49,14 @@ public class MockApplicationPackage implements ApplicationPackage { public static final String DEPLOYED_BY_USER = "user"; public static final String APPLICATION_NAME = "application"; public static final long APPLICATION_GENERATION = 1L; - public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo"); - public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar"); + public static final String MUSIC_SCHEMA = createSchema("music", "foo"); + public static final String BOOK_SCHEMA = createSchema("book", "bar"); private final File root; private final String hostsS; private final String servicesS; private final List<String> schemas; + private final Map<Path, MockApplicationFile> files; private final String schemaDir; private final Optional<String> deploymentSpec; private final Optional<String> validationOverrides; @@ -68,6 +65,7 @@ public class MockApplicationPackage implements ApplicationPackage { private final ApplicationMetaData applicationMetaData; protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas, + Map<Path, MockApplicationFile> files, String schemaDir, String deploymentSpec, String validationOverrides, boolean failOnValidateXml, String queryProfile, String queryProfileType) { @@ -75,6 +73,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.hostsS = hosts; this.servicesS = services; this.schemas = schemas; + this.files = files; this.schemaDir = schemaDir; this.deploymentSpec = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); @@ -135,7 +134,7 @@ public class MockApplicationPackage implements ApplicationPackage { throw new IllegalArgumentException("Expected the first line of a schema but got '" + sd + "'"); int end = s.indexOf(' '); if (end < 0) - end = s.indexOf('}'); + end = s.indexOf('}'); return s.substring(0, end).trim(); } @@ -165,6 +164,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile getFile(Path file) { + if (files.containsKey(file)) return files.get(file); return new MockApplicationFile(file, Path.fromString(root.toString())); } @@ -239,6 +239,7 @@ public class MockApplicationPackage implements ApplicationPackage { private String hosts = null; private String services = null; private List<String> schemas = Collections.emptyList(); + private Map<Path, MockApplicationFile> files = new LinkedHashMap<>(); private String schemaDir = null; private String deploymentSpec = null; private String validationOverrides = null; @@ -282,6 +283,16 @@ public class MockApplicationPackage implements ApplicationPackage { return this; } + /** Additional (mock) files that will exist in this application package, with their content. */ + public Builder withFiles(Map<Path, String> files) { + Map<Path, MockApplicationFile> mockFiles = new HashMap<>(); + for (var file : files.entrySet()) + mockFiles.put(file.getKey(), new MockApplicationFile(file.getKey(), + Path.fromString(root.toString()), file.getValue())); + this.files = mockFiles; + return this; + } + public Builder withSchemaDir(String schemaDir) { this.schemaDir = schemaDir; return this; @@ -313,13 +324,13 @@ public class MockApplicationPackage implements ApplicationPackage { } public ApplicationPackage build() { - return new MockApplicationPackage(root, hosts, services, schemas, schemaDir, - deploymentSpec, validationOverrides, failOnValidateXml, - queryProfile, queryProfileType); + return new MockApplicationPackage(root, hosts, services, schemas, files, schemaDir, + deploymentSpec, validationOverrides, failOnValidateXml, + queryProfile, queryProfileType); } } - public static String createSearchDefinition(String name, String fieldName) { + public static String createSchema(String name, String fieldName) { return "search " + name + " {" + " document " + name + " {" + " field " + fieldName + " type string {}" + @@ -369,25 +380,36 @@ public class MockApplicationPackage implements ApplicationPackage { /** The File pointing to the actual file represented by this */ private final File file; + /** The content of this file, or null to read it from the file system. */ + private final String content; + public MockApplicationFile(Path filePath, Path applicationPackagePath) { + this(filePath, applicationPackagePath, null); + } + + private MockApplicationFile(Path filePath, Path applicationPackagePath, String content) { super(filePath); this.root = applicationPackagePath; file = applicationPackagePath.append(filePath).toFile(); + this.content = content; } @Override public boolean isDirectory() { + if (content != null) return false; return file.isDirectory(); } @Override public boolean exists() { + if (content != null) return true; return file.exists(); } @Override public Reader createReader() { try { + if (content != null) return new StringReader(content); if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); return IOUtils.createReader(file, "UTF-8"); } @@ -399,6 +421,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public InputStream createInputStream() { try { + if (content != null) throw new UnsupportedOperationException("Not implemented for mock file content"); if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); return new BufferedInputStream(new FileInputStream(file)); } @@ -416,6 +439,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile writeFile(Reader input) { try { + if (content != null) throw new UnsupportedOperationException("Not implemented for mock file content"); IOUtils.writeFile(file, IOUtils.readAll(input), false); return this; } @@ -427,6 +451,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile appendFile(String value) { try { + if (content != null) throw new UnsupportedOperationException("Not implemented for mock file content"); IOUtils.writeFile(file, value, true); return this; } @@ -437,7 +462,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List<ApplicationFile> listFiles(PathFilter filter) { - if ( ! isDirectory()) return Collections.emptyList(); + if ( ! isDirectory()) return List.of(); return Arrays.stream(file.listFiles()).filter(f -> filter.accept(Path.fromString(f.toString()))) .map(f -> new MockApplicationFile(asApplicationRelativePath(f), root)) .collect(Collectors.toList()); @@ -445,6 +470,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile delete() { + if (content != null) throw new UnsupportedOperationException("Not implemented for mock file content"); file.delete(); return this; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java index 377e19eedf2..7a2765de852 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java @@ -12,6 +12,7 @@ import com.yahoo.tensor.TensorType; * @author bratseth */ public class RankingConstant extends DistributableResource { + private TensorType tensorType = null; public RankingConstant(String name) { @@ -43,10 +44,9 @@ public class RankingConstant extends DistributableResource { ": Dense tensor dimensions must have a size"); } + @Override public String toString() { - StringBuilder b = new StringBuilder(super.toString()) - .append("' of type '").append(tensorType).append("'"); - return b.toString(); + return super.toString() + "' of type '" + tensorType + "'"; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java index 5020e9a061c..339840f66ce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java @@ -5,14 +5,14 @@ import com.yahoo.config.application.api.FileRegistry; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; /** - * Constant values for ranking/model execution tied to a search definition, or globally to an application - * package + * Constant values for ranking/model execution tied to a schema, or globally to an application package * * @author bratseth */ @@ -23,7 +23,7 @@ public class RankingConstants { /** The schema this belongs to, or empty if it is global */ private final Optional<Schema> owner; - private final Map<String, RankingConstant> constants = new ConcurrentHashMap<>(); + private final Map<String, RankingConstant> constants = new LinkedHashMap<>(); public RankingConstants(FileRegistry fileRegistry, Optional<Schema> owner) { this.fileRegistry = fileRegistry; @@ -36,7 +36,7 @@ public class RankingConstants { String name = constant.getName(); RankingConstant prev = constants.putIfAbsent(name, constant); if ( prev != null ) - throw new IllegalArgumentException("Ranking constant '" + name + "' defined twice"); + throw new IllegalArgumentException("Constant '" + name + "' defined twice"); } public void putIfAbsent(RankingConstant constant) { @@ -70,7 +70,7 @@ public class RankingConstants { if (owner.isEmpty() || owner.get().inherited().isEmpty()) return Collections.unmodifiableMap(constants); if (constants.isEmpty()) return owner.get().inherited().get().rankingConstants().asMap(); - var allConstants = new HashMap<>(owner.get().inherited().get().rankingConstants().asMap()); + var allConstants = new LinkedHashMap<>(owner.get().inherited().get().rankingConstants().asMap()); allConstants.putAll(constants); return Collections.unmodifiableMap(allConstants); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java index 4c9e65f16c8..c36bd96d309 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -50,7 +50,7 @@ public class RankProfileList extends Derived implements RankProfilesConfig.Produ } /** - * Creates a rank profile + * Creates a rank profile list * * @param schema the schema this is a rank profile from * @param attributeFields the attribute fields to create a ranking for diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java index 9c2d29282ae..d3c24fa9924 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java @@ -80,7 +80,7 @@ public class ConvertParsedSchemas { this.typeConverter = new ConvertParsedTypes(orderedInput, docMan); } - private Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap(); + private final Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap<>(); public List<Schema> convertToSchemas() { typeConverter.convert(false); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java index 8f25a586aa6..e46a6882bc7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java @@ -118,7 +118,7 @@ public class ConvertSchemaCollection { } private void resolveStructInheritance() { - List<ParsedStruct> all = new ArrayList(); + List<ParsedStruct> all = new ArrayList<>(); for (var schema : orderedInput) { var doc = schema.getDocument(); for (var struct : doc.getStructs()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java index 599dd6e2a7a..e4f312e98e3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java @@ -14,9 +14,12 @@ import java.util.Optional; /** * This class holds the extracted information after parsing * one schema (.sd) file, using simple data structures - * as far as possible. Do not put advanced logic here! + * as far as possible. + * + * Do not put advanced logic here! + * * @author arnej27959 - **/ + */ public class ParsedSchema extends ParsedBlock { public static class ImportedField { @@ -39,8 +42,8 @@ public class ParsedSchema extends ParsedBlock { private final List<RankingConstant> rankingConstants = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); private final List<String> inheritedByDocument = new ArrayList<>(); - private final Map<String, ParsedSchema> resolvedInherits = new LinkedHashMap(); - private final Map<String, ParsedSchema> allResolvedInherits = new LinkedHashMap(); + private final Map<String, ParsedSchema> resolvedInherits = new LinkedHashMap<>(); + private final Map<String, ParsedSchema> allResolvedInherits = new LinkedHashMap<>(); private final Map<String, ParsedAnnotation> extraAnnotations = new LinkedHashMap<>(); private final Map<String, ParsedDocumentSummary> docSums = new LinkedHashMap<>(); private final Map<String, ParsedField> extraFields = new LinkedHashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index 82c793af70a..c2f71c78ce3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -66,15 +66,15 @@ public class RankingConstantsValidator extends Validator { // TODO: Handle validation of URI soon too. if (rankingConstant.getPathType() == RankingConstant.PathType.FILE) { String constantFile = rankingConstant.getFileName(); - if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) && - constantFile.startsWith(FilesApplicationPackage.preprocessed)) { + if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) + && constantFile.startsWith(FilesApplicationPackage.preprocessed)) { constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length()); } ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile)); new ConstantTensorJsonValidator().validate(constantFile, - rankingConstant.getTensorType(), - tensorApplicationFile.createReader()); + rankingConstant.getTensorType(), + tensorApplicationFile.createReader()); } } diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index 887bdb7f789..1fabe9c57c4 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -1768,7 +1768,7 @@ void rankingConstant(ParsedSchema schema) : { ( <CONSTANT> name = identifier() lbrace() (path = fileItem() { pathType = DistributableResource.PathType.FILE; } - | path = uriItem() { pathType = DistributableResource.PathType.URI; } // Undocumented. TODO: Remove oin Vespa 8 + | path = uriItem() { pathType = DistributableResource.PathType.URI; } // Undocumented. TODO: Remove on Vespa 8 | type = tensorTypeWithPrefix(rankingConstantErrorMessage(name)) (<NL>)* )+ <RBRACE> diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java index 95fcd8d7832..0929e54a360 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java @@ -25,6 +25,7 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -389,8 +390,8 @@ public class RankingExpressionWithOnnxTestCase { StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { super(new File(applicationPackageWritableRoot.toString()), - null, null, Collections.emptyList(), null, - null, null, false, queryProfile, queryProfileType); + null, null, List.of(), Map.of(), null, + null, null, false, queryProfile, queryProfileType); } @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index 666aef6df73..a7f3d134b57 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -23,8 +23,8 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.List; -import static com.yahoo.config.model.test.MockApplicationPackage.BOOK_SEARCHDEFINITION; -import static com.yahoo.config.model.test.MockApplicationPackage.MUSIC_SEARCHDEFINITION; +import static com.yahoo.config.model.test.MockApplicationPackage.BOOK_SCHEMA; +import static com.yahoo.config.model.test.MockApplicationPackage.MUSIC_SCHEMA; /** * @author bratseth @@ -77,7 +77,7 @@ public class ValidationTester { Provisioned provisioned = hostProvisioner.startProvisionedRecording(); ApplicationPackage newApp = new MockApplicationPackage.Builder() .withServices(services) - .withSchemas(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION)) + .withSchemas(ImmutableList.of(MUSIC_SCHEMA, BOOK_SCHEMA)) .withValidationOverrides(validationOverrides) .build(); VespaModelCreatorWithMockPkg newModelCreator = new VespaModelCreatorWithMockPkg(newApp); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index fa6fc1244d8..567534ba9f6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import java.time.Instant; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index b05404ec08f..3468d81b748 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -32,7 +32,6 @@ import java.util.List; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.config.search.core.ProtonConfig.Feeding.Shared_field_writer_executor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -770,7 +769,7 @@ public class ContentBuilderTest extends DomBuilderTest { DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true)); VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withServices(hostedXml) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build()) .create(deployStateBuilder); ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); @@ -797,7 +796,7 @@ public class ContentBuilderTest extends DomBuilderTest { var deployStateBuilder = new DeployState.Builder().properties(props); var model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withServices(hostedXml) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build()) .create(deployStateBuilder); return getProtonConfig(model.getContentClusters().values().iterator().next()); @@ -866,7 +865,7 @@ public class ContentBuilderTest extends DomBuilderTest { return new MockApplicationPackage.Builder() .withHosts(hosts) .withServices(services) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build(); } @@ -887,7 +886,7 @@ public class ContentBuilderTest extends DomBuilderTest { VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withHosts(getHosts()) .withServices(combined) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build()) .create(deployStateBuilder); @@ -907,8 +906,8 @@ public class ContentBuilderTest extends DomBuilderTest { VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withHosts(getHosts()) .withServices(combined) - .withSchemas(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION, - MockApplicationPackage.BOOK_SEARCHDEFINITION)) + .withSchemas(Arrays.asList(MockApplicationPackage.MUSIC_SCHEMA, + MockApplicationPackage.BOOK_SCHEMA)) .build()) .create(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java index 44ca69d55c8..d48c397b1f0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java @@ -54,7 +54,7 @@ public class ImplicitIndexingClusterTest { return new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withServices("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + servicesXml) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build()) .create(deployStateBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 671a9f6660d..059c1ceb208 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -4,12 +4,15 @@ package com.yahoo.vespa.model.search.test; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.vespa.config.search.IndexschemaConfig; -import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.path.Path; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.IndexschemaConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.ContentSearchCluster; @@ -17,10 +20,11 @@ import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.search.IndexedSearchCluster; import org.junit.Test; -import java.util.List; import java.util.Arrays; -import java.util.Map; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -94,7 +98,7 @@ public class DocumentDatabaseTestCase { var tester = new SchemaTester(); VespaModel model = tester.createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties)); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); assertEquals(global, proton.feeding().concurrency(), SMALL); assertEquals(local.size(), proton.documentdb().size()); for (int i = 0; i < local.size(); i++) { @@ -109,7 +113,7 @@ public class DocumentDatabaseTestCase { DocType.create("b", "streaming"), DocType.create("c", "store-only")), ""); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals(ProtonConfig.Documentdb.Mode.Enum.INDEX, proton.documentdb(0).mode()); assertEquals("a", proton.documentdb(0).inputdoctypename()); @@ -124,7 +128,7 @@ public class DocumentDatabaseTestCase { assertEquals(nameAndModes.size(), local.size()); VespaModel model = tester.createModel(nameAndModes, xmlTuning); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); assertEquals(local.size(), proton.documentdb().size()); for (int i = 0; i < local.size(); i++) { assertEquals(local.get(i).longValue(), proton.documentdb(i).allocation().initialnumdocs()); @@ -171,7 +175,7 @@ public class DocumentDatabaseTestCase { String type3Id = "test/search/cluster.test/type3"; { assertEquals(3, indexedSearchCluster.getDocumentDbs().size()); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals("type1", proton.documentdb(0).inputdoctypename()); assertEquals(type1Id, proton.documentdb(0).configid()); @@ -210,6 +214,43 @@ public class DocumentDatabaseTestCase { } @Test + public void testRankingConstants() { + List<String> schemas = List.of("type1"); + var tester = new SchemaTester(); + + // Use lz4 endings to avoid having to provide file content to be validated + String schemaConstants = + " constant constant_1 {" + + " file: constants/my_constant_1.json.lz4" + + " type: tensor<float>(x{},y{})" + + " }" + + " constant constant_2 {" + + " file: constants/my_constant_2.json.lz4" + + " type: tensor(x[1000])" + + " }"; + + Map<Path, String> constants = new HashMap<>(); + constants.put(Path.fromString("constants/my_constant_1.json.lz4"), ""); + constants.put(Path.fromString("constants/my_constant_2.json.lz4"), ""); + var model = tester.createModel(schemaConstants, "", schemas, constants); + IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); + RankingConstantsConfig.Builder b = new RankingConstantsConfig.Builder(); + indexedSearchCluster.getDocumentDbs().get(0).getConfig(b); + RankingConstantsConfig config = b.build(); + assertEquals(2, config.constant().size()); + + var constant1Config = config.constant().get(0); + assertEquals("constant_1", constant1Config.name()); + assertEquals("constants/my_constant_1.json.lz4", constant1Config.fileref().value()); + assertEquals("tensor<float>(x{},y{})", constant1Config.type()); + + var constant2Config = config.constant().get(1); + assertEquals("constant_2", constant2Config.name()); + assertEquals("constants/my_constant_2.json.lz4", constant2Config.fileref().value()); + assertEquals("tensor(x[1000])", constant2Config.type()); + } + + @Test public void requireThatRelevantConfigIsAvailableForClusterSearcher() { String inputsProfile = " rank-profile inputs {" + @@ -300,7 +341,7 @@ public class DocumentDatabaseTestCase { var model = tester.createModelWithMode(mode, sds, builder); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); assertEquals(sds.size(), proton.documentdb().size()); for (int i = 0; i < sds.size(); i++) { assertEquals(sds.get(i), proton.documentdb(i).inputdoctypename()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java index 7a077c1a0aa..6b740816775 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.model.search.test; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.path.Path; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -14,6 +16,7 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -72,21 +75,22 @@ public class SchemaTester { return retval.toString(); } - ProtonConfig getProtonCfg(ContentSearchCluster cluster) { + ProtonConfig getProtonConfig(ContentSearchCluster cluster) { ProtonConfig.Builder pb = new ProtonConfig.Builder(); cluster.getConfig(pb); return new ProtonConfig(pb); } void assertSingleSD(String mode) { - List<String> sds = List.of("type1"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - generateSchemas("", sds)).create(); + List<String> schemas = List.of("type1"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, mode), + generateSchemas("", "", schemas), + Map.of()).create(); IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); String type1Id = "test/search/cluster.test/type1"; - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = getProtonConfig(contentSearchCluster); assertEquals(1, proton.documentdb().size()); assertEquals("type1", proton.documentdb(0).inputdoctypename()); assertEquals(type1Id, proton.documentdb(0).configid()); @@ -95,13 +99,29 @@ public class SchemaTester { VespaModel createModel(List<String> schemas) { return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - generateSchemas("", schemas)).create(); + generateSchemas("", "", schemas), + Map.of()).create(); } VespaModel createModelWithRankProfile(String rankProfile, List<String> schemas) { return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - generateSchemas(rankProfile, schemas)).create(); + generateSchemas("", rankProfile, schemas), + Map.of()).create(); + } + + VespaModel createModelWithSchemaContent(String schemaContent, List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas(schemaContent, "", schemas), + Map.of()).create(); + } + + VespaModel createModel(String schemaContent, String rankProfile, List<String> schemas, Map<Path, String> files) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas(schemaContent, rankProfile, schemas), + files).create(); } VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) { @@ -111,27 +131,29 @@ public class SchemaTester { VespaModel createModelWithMode(String mode, List<String> schemas) { return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, mode), - generateSchemas("", schemas)).create(); + generateSchemas("", "", schemas), + Map.of()).create(); } VespaModel createModelWithMode(String mode, List<String> schemas, DeployState.Builder builder) { return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, mode), - generateSchemas("", schemas)).create(builder); + generateSchemas("", "", schemas), + Map.of()).create(builder); } VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) { - List<String> sds = new ArrayList<>(nameAndModes.size()); - for (DocType nameAndMode : nameAndModes) { - sds.add(nameAndMode.getType()); - } + List<String> schemas = new ArrayList<>(nameAndModes.size()); + for (DocType nameAndMode : nameAndModes) + schemas.add(nameAndMode.getType()); var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), - generateSchemas("", sds)); + generateSchemas("", "", schemas), + Map.of()); return builder != null ? creator.create(builder) : creator.create(); } - public static String generateSchema(String name, String field1, String field2, String rankProfile) { + public static String generateSchema(String name, String field1, String field2, String schemaContent, String rankProfile) { return "schema " + name + " {" + " document " + name + " {" + " field " + field1 + " type string {\n" + @@ -146,6 +168,7 @@ public class SchemaTester { " indexing: attribute \n" + " }\n" + " }\n" + + schemaContent + " rank-profile staticrank inherits default {" + " first-phase { expression: attribute(" + field2 + ") }" + " }" + @@ -163,18 +186,18 @@ public class SchemaTester { "}"; } - public static List<String> generateSchemas(String rankProfile, String ... sdNames) { - return generateSchemas(rankProfile, Arrays.asList(sdNames)); + public static List<String> generateSchemas(String schemaContent, String rankProfile, String ... schemaNames) { + return generateSchemas(schemaContent, rankProfile, Arrays.asList(schemaNames)); } - public static List<String> generateSchemas(String rankProfile, List<String> sdNames) { - List<String> sds = new ArrayList<>(); + public static List<String> generateSchemas(String schemaContent, String rankProfile, List<String> schemaNames) { + List<String> schemas = new ArrayList<>(); int i = 0; - for (String sdName : sdNames) { - sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2), rankProfile)); + for (String sdName : schemaNames) { + schemas.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2), schemaContent, rankProfile)); i = i + 2; } - return sds; + return schemas; } DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index d777fefbbe5..2c70c7b2da5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -11,10 +11,12 @@ import com.yahoo.config.model.api.ValidationParameters.CheckRouting; import com.yahoo.config.model.application.provider.SchemaValidators; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.path.Path; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validation; import java.util.List; +import java.util.Map; /** * For testing purposes only. @@ -32,7 +34,15 @@ public class VespaModelCreatorWithMockPkg { } public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> schemas) { - this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSchemas(schemas).build()); + this(hosts, services, schemas, Map.of()); + } + + public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> schemas, Map<Path, String> files) { + this(new MockApplicationPackage.Builder().withHosts(hosts) + .withServices(services) + .withSchemas(schemas) + .withFiles(files) + .build()); } public VespaModelCreatorWithMockPkg(ApplicationPackage appPkg) { |